refactor: implement a proper code migration system (#324)

* refactor: implement a proper code migration system

* chore: rename .tsx -> .ts

app/server/core/constants.ts

@@ -5,5 +5,3 @@ export const DATABASE_URL = process.env.DATABASE_URL || "/var/lib/zerobyte/data/
 export const RESTIC_PASS_FILE = "/var/lib/zerobyte/data/restic.pass";
 
 export const DEFAULT_EXCLUDES = [DATABASE_URL, RESTIC_PASS_FILE, REPOSITORY_BASE];
-
-export const REQUIRED_MIGRATIONS = []; // ["v0.21.1"] add this once re-tagging migration is removed

app/server/index.ts

@@ -2,14 +2,12 @@ import { createHonoServer } from "react-router-hono-server/bun";
 import * as schema from "./db/schema";
 import { setSchema, runDbMigrations } from "./db/db";
 import { startup } from "./modules/lifecycle/startup";
-import { retagSnapshots } from "./modules/lifecycle/migration";
 import { logger } from "./utils/logger";
 import { shutdown } from "./modules/lifecycle/shutdown";
-import { REQUIRED_MIGRATIONS } from "./core/constants";
-import { validateRequiredMigrations } from "./modules/lifecycle/checkpoint";
 import { createApp } from "./app";
 import { config } from "./core/config";
 import { runCLI } from "./cli";
+import { runMigrations } from "./modules/lifecycle/migrations";
 
 setSchema(schema);
 
@@ -22,9 +20,7 @@ const app = createApp();
 
 runDbMigrations();
 
-await retagSnapshots();
+await runMigrations();
-await validateRequiredMigrations(REQUIRED_MIGRATIONS);
-
 await startup();
 
 export type AppType = typeof app;

app/server/modules/lifecycle/checkpoint.ts

@@ -1,88 +0,0 @@
-import { eq, sql } from "drizzle-orm";
-import { db } from "../../db/db";
-import { appMetadataTable, usersTable } from "../../db/schema";
-import { logger } from "../../utils/logger";
-
-const MIGRATION_KEY_PREFIX = "migration:";
-
-export const recordMigrationCheckpoint = async (version: string): Promise<void> => {
-	const key = `${MIGRATION_KEY_PREFIX}${version}`;
-	const now = Date.now();
-
-	await db
-		.insert(appMetadataTable)
-		.values({
-			key,
-			value: JSON.stringify({ completedAt: new Date().toISOString() }),
-			createdAt: now,
-			updatedAt: now,
-		})
-		.onConflictDoUpdate({
-			target: appMetadataTable.key,
-			set: {
-				value: JSON.stringify({ completedAt: new Date().toISOString() }),
-				updatedAt: now,
-			},
-		});
-
-	logger.info(`Recorded migration checkpoint for ${version}`);
-};
-
-export const hasMigrationCheckpoint = async (version: string): Promise<boolean> => {
-	const key = `${MIGRATION_KEY_PREFIX}${version}`;
-	const result = await db.query.appMetadataTable.findFirst({
-		where: eq(appMetadataTable.key, key),
-	});
-	return result !== undefined;
-};
-
-export const validateRequiredMigrations = async (requiredVersions: string[]): Promise<void> => {
-	const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
-	const isFreshInstall = userCount[0]?.count === 0;
-
-	if (isFreshInstall) {
-		logger.info("Fresh installation detected, skipping migration checkpoint validation.");
-
-		for (const version of requiredVersions) {
-			const hasCheckpoint = await hasMigrationCheckpoint(version);
-			if (!hasCheckpoint) {
-				await recordMigrationCheckpoint(version);
-			}
-		}
-
-		return;
-	}
-
-	for (const version of requiredVersions) {
-		const hasCheckpoint = await hasMigrationCheckpoint(version);
-		if (!hasCheckpoint) {
-			logger.error(`
-================================================================================
-MIGRATION ERROR: Required migration ${version} has not been run.
-
-You are attempting to start a version of Zerobyte that requires migration
-checkpoints from previous versions. This typically happens when you skip
-versions during an upgrade.
-
-To fix this:
-1. First upgrade to version ${version} and run the application once
-2. Validate that everything is still working correctly
-3. Then upgrade to the current version
-
-================================================================================
-`);
-			process.exit(1);
-		}
-	}
-};
-
-export const getMigrationCheckpoints = async (): Promise<{ version: string; completedAt: string }[]> => {
-	const results = await db.query.appMetadataTable.findMany({
-		where: (table, { like }) => like(table.key, `${MIGRATION_KEY_PREFIX}%`),
-	});
-
-	return results.map((r) => ({
-		version: r.key.replace(MIGRATION_KEY_PREFIX, ""),
-		completedAt: JSON.parse(r.value).completedAt,
-	}));
-};

app/server/modules/lifecycle/migrations.ts

@@ -0,0 +1,114 @@
+import { db } from "~/server/db/db";
+import { logger } from "../../utils/logger";
+import { v00001 } from "./migrations/00001-retag-snapshots";
+import { usersTable } from "~/server/db/schema";
+import { sql } from "drizzle-orm";
+import { eq } from "drizzle-orm";
+import { appMetadataTable } from "../../db/schema";
+
+const MIGRATION_KEY_PREFIX = "migration:";
+
+const recordMigrationCheckpoint = async (version: string): Promise<void> => {
+	const key = `${MIGRATION_KEY_PREFIX}${version}`;
+	const now = Date.now();
+
+	await db
+		.insert(appMetadataTable)
+		.values({ key, value: JSON.stringify({ completedAt: new Date().toISOString() }), createdAt: now, updatedAt: now })
+		.onConflictDoUpdate({
+			target: appMetadataTable.key,
+			set: { value: JSON.stringify({ completedAt: new Date().toISOString() }), updatedAt: now },
+		});
+
+	logger.info(`Recorded migration checkpoint for ${version}`);
+};
+
+const hasMigrationCheckpoint = async (id: string): Promise<boolean> => {
+	const key = `${MIGRATION_KEY_PREFIX}${id}`;
+	const result = await db.query.appMetadataTable.findFirst({
+		where: eq(appMetadataTable.key, key),
+	});
+	return result !== undefined;
+};
+
+type MigrationEntity = {
+	execute: () => Promise<{ success: boolean; errors: Array<{ name: string; error: string }> }>;
+	id: string;
+	type: "maintenance" | "critical";
+	dependsOn?: string[];
+};
+
+const registry: MigrationEntity[] = [v00001];
+
+export const runMigrations = async () => {
+	const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
+	const isFreshInstall = userCount[0]?.count === 0;
+
+	if (isFreshInstall) {
+		logger.debug("Fresh installation detected, skipping migration checkpoint validation.");
+
+		for (const migration of registry) {
+			const hasCheckpoint = await hasMigrationCheckpoint(migration.id);
+			if (!hasCheckpoint) {
+				await recordMigrationCheckpoint(migration.id);
+			}
+		}
+
+		return;
+	}
+
+	for (const migration of registry) {
+		const alreadyMigrated = await hasMigrationCheckpoint(migration.id);
+
+		if (alreadyMigrated) {
+			logger.debug(`Migration ${migration.id} already completed, skipping.`);
+			continue;
+		}
+
+		if (migration.dependsOn) {
+			for (const dep of migration.dependsOn) {
+				const depCompleted = await hasMigrationCheckpoint(dep);
+				if (!depCompleted) {
+					const err = [
+						"================================================================================",
+						`🚨 MIGRATION ERROR: Migration ${migration.id} depends on migration ${dep}.`,
+						"The application cannot start until the required migration has successfully completed.",
+						"Please fix the issues and restart the application.",
+						"",
+						"Seek support by opening an issue on the Zerobyte GitHub repository if you need assistance.",
+						"================================================================================",
+					];
+					err.forEach((line) => logger.error(line));
+					process.exit(1);
+				}
+			}
+		}
+
+		logger.info(`Running migration: ${migration.id} (${migration.type})`);
+		const result = await migration.execute();
+		if (result.success) {
+			logger.info(`Migration ${migration.id} completed successfully.`);
+			await recordMigrationCheckpoint(migration.id);
+		} else {
+			logger.error(`Migration ${migration.id} completed with errors: ${result.errors.length} items failed.`);
+			for (const err of result.errors) {
+				logger.error(`Migration failure - ${err.name}: ${err.error}`);
+			}
+
+			if (migration.type === "critical") {
+				const err = [
+					"================================================================================",
+					`🚨 MIGRATION ERROR: Critical migration ${migration.id} failed.`,
+					"",
+					"The application cannot start until this migration has successfully completed.",
+					"",
+					"Please fix the issues and restart the application. Seek support by opening an issue",
+					"on the Zerobyte GitHub repository if you need assistance.",
+					"================================================================================",
+				];
+				err.forEach((line) => logger.error(line));
+				process.exit(1);
+			}
+		}
+	}
+};

app/server/modules/lifecycle/migrations/00001-retag-snapshots.ts

@@ -1,60 +1,11 @@
 import { eq } from "drizzle-orm";
-import { db } from "../../db/db";
+import { db } from "../../../db/db";
-import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../db/schema";
+import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../../db/schema";
-import { logger } from "../../utils/logger";
+import { logger } from "../../../utils/logger";
-import { hasMigrationCheckpoint, recordMigrationCheckpoint } from "./checkpoint";
 import { toMessage } from "~/server/utils/errors";
 import { safeSpawn } from "~/server/utils/spawn";
 import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "~/server/utils/restic";
 
-const MIGRATION_VERSION = "v0.21.1";
-
-interface MigrationResult {
-	success: boolean;
-	errors: Array<{ name: string; error: string }>;
-}
-
-export class MigrationError extends Error {
-	version: string;
-	failedItems: Array<{ name: string; error: string }>;
-
-	constructor(version: string, failedItems: Array<{ name: string; error: string }>) {
-		const itemNames = failedItems.map((e) => e.name).join(", ");
-		super(`Migration ${version} failed for: ${itemNames}`);
-		this.version = version;
-		this.failedItems = failedItems;
-		this.name = "MigrationError";
-	}
-}
-
-export const retagSnapshots = async () => {
-	const alreadyMigrated = await hasMigrationCheckpoint(MIGRATION_VERSION);
-	if (alreadyMigrated) {
-		logger.debug(`Migration ${MIGRATION_VERSION} already completed, skipping.`);
-		return;
-	}
-
-	logger.info(`Starting snapshots retagging migration (${MIGRATION_VERSION})...`);
-
-	const result = await migrateSnapshotsToShortIdTag();
-	const allErrors = [...result.errors];
-
-	if (allErrors.length > 0) {
-		logger.error(`Migration ${MIGRATION_VERSION} completed with errors: ${allErrors.length} items failed.`);
-		logger.error(
-			`Some snapshots could not be retagged. Please check the logs for details. Fix any repository in error state and re-start zerobyte to retry the migration for failed items.`,
-		);
-		for (const err of allErrors) {
-			logger.error(`Migration failure - ${err.name}: ${err.error}`);
-		}
-
-		return;
-	}
-
-	await recordMigrationCheckpoint(MIGRATION_VERSION);
-	logger.info(`Snapshots retagging migration (${MIGRATION_VERSION}) complete.`);
-};
-
 const migrateTag = async (
 	oldTag: string,
 	newTag: string,
@@ -81,7 +32,7 @@ const migrateTag = async (
 	return null;
 };
 
-const migrateSnapshotsToShortIdTag = async (): Promise<MigrationResult> => {
+const execute = async () => {
 	const errors: Array<{ name: string; error: string }> = [];
 	const backupSchedules = await db.query.backupSchedulesTable.findMany({});
 
@@ -134,3 +85,9 @@ const migrateSnapshotsToShortIdTag = async (): Promise<MigrationResult> => {
 
 	return { success: errors.length === 0, errors };
 };
+
+export const v00001 = {
+	execute,
+	id: "00001-retag-snapshots",
+	type: "maintenance" as const,
+};

app/server/utils/logger.ts

@@ -27,7 +27,7 @@ const log = (level: "info" | "warn" | "error" | "debug", messages: unknown[]) =>
 			return sanitizeSensitiveData(JSON.stringify(m, null, 2));
 		}
 
-		return sanitizeSensitiveData(String(JSON.stringify(m)));
+		return sanitizeSensitiveData(String(m as string));
 	});
 
 	winstonLogger.log(level, stringMessages.join(" "));