refactor(startup): ensure entities always use the latest configuration format (#173)

* refactor(startup): ensure entities always use the latest configuration schema

* refactor: await config updates to avoid race condition on later mount

app/client/modules/notifications/components/create-notification-form.tsx

@@ -17,11 +17,11 @@ import { Input } from "~/client/components/ui/input";
 import { SecretInput } from "~/client/components/ui/secret-input";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "~/client/components/ui/select";
 import { Checkbox } from "~/client/components/ui/checkbox";
-import { notificationConfigSchema } from "~/schemas/notifications";
+import { notificationConfigSchemaBase } from "~/schemas/notifications";
 
 export const formSchema = type({
 	name: "2<=string<=32",
-}).and(notificationConfigSchema);
+}).and(notificationConfigSchemaBase);
 const cleanSchema = type.pipe((d) => formSchema(deepClean(d)));
 
 export type NotificationFormValues = typeof formSchema.inferIn;

app/client/modules/repositories/components/create-repository-form.tsx

@@ -20,7 +20,7 @@ import { SecretInput } from "../../../components/ui/secret-input";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../../../components/ui/select";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../../../components/ui/tooltip";
 import { useSystemInfo } from "~/client/hooks/use-system-info";
-import { COMPRESSION_MODES, repositoryConfigSchema } from "~/schemas/restic";
+import { COMPRESSION_MODES, repositoryConfigSchemaBase } from "~/schemas/restic";
 import { Checkbox } from "../../../components/ui/checkbox";
 import {
 	LocalRepositoryForm,
@@ -36,7 +36,7 @@ import {
 export const formSchema = type({
 	name: "2<=string<=32",
 	compressionMode: type.valueOf(COMPRESSION_MODES).optional(),
-}).and(repositoryConfigSchema);
+}).and(repositoryConfigSchemaBase);
 const cleanSchema = type.pipe((d) => formSchema(deepClean(d)));
 
 export type RepositoryFormValues = typeof formSchema.inferIn;

app/client/modules/volumes/components/create-volume-form.tsx

@@ -18,7 +18,7 @@ import {
 } from "../../../components/ui/form";
 import { Input } from "../../../components/ui/input";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../../../components/ui/select";
-import { volumeConfigSchema } from "~/schemas/volumes";
+import { volumeConfigSchemaBase } from "~/schemas/volumes";
 import { testConnectionMutation } from "../../../api-client/@tanstack/react-query.gen";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../../../components/ui/tooltip";
 import { useSystemInfo } from "~/client/hooks/use-system-info";
@@ -26,7 +26,7 @@ import { DirectoryForm, NFSForm, SMBForm, WebDAVForm, RcloneForm } from "./volum
 
 export const formSchema = type({
 	name: "2<=string<=32",
-}).and(volumeConfigSchema);
+}).and(volumeConfigSchemaBase);
 const cleanSchema = type.pipe((d) => formSchema(deepClean(d)));
 
 export type FormValues = typeof formSchema.inferIn;

app/schemas/notifications.ts

@@ -76,7 +76,7 @@ export const customNotificationConfigSchema = type({
 	shoutrrrUrl: "string",
 });
 
-export const notificationConfigSchema = emailNotificationConfigSchema
+export const notificationConfigSchemaBase = emailNotificationConfigSchema
 	.or(slackNotificationConfigSchema)
 	.or(discordNotificationConfigSchema)
 	.or(gotifyNotificationConfigSchema)
@@ -85,6 +85,8 @@ export const notificationConfigSchema = emailNotificationConfigSchema
 	.or(telegramNotificationConfigSchema)
 	.or(customNotificationConfigSchema);
 
+export const notificationConfigSchema = notificationConfigSchemaBase.onUndeclaredKey("delete");
+
 export type NotificationConfig = typeof notificationConfigSchema.infer;
 
 export const NOTIFICATION_EVENTS = {

app/schemas/restic.ts

@@ -79,7 +79,7 @@ export const sftpRepositoryConfigSchema = type({
 	privateKey: "string",
 }).and(baseRepositoryConfigSchema);
 
-export const repositoryConfigSchema = s3RepositoryConfigSchema
+export const repositoryConfigSchemaBase = s3RepositoryConfigSchema
 	.or(r2RepositoryConfigSchema)
 	.or(localRepositoryConfigSchema)
 	.or(gcsRepositoryConfigSchema)
@@ -88,6 +88,8 @@ export const repositoryConfigSchema = s3RepositoryConfigSchema
 	.or(restRepositoryConfigSchema)
 	.or(sftpRepositoryConfigSchema);
 
+export const repositoryConfigSchema = repositoryConfigSchemaBase.onUndeclaredKey("delete");
+
 export type RepositoryConfig = typeof repositoryConfigSchema.infer;
 
 export const COMPRESSION_MODES = {

app/schemas/volumes.ts

@@ -55,7 +55,13 @@ export const rcloneConfigSchema = type({
 	readOnly: "boolean?",
 });
 
-export const volumeConfigSchema = nfsConfigSchema.or(smbConfigSchema).or(webdavConfigSchema).or(directoryConfigSchema).or(rcloneConfigSchema);
+export const volumeConfigSchemaBase = nfsConfigSchema
+	.or(smbConfigSchema)
+	.or(webdavConfigSchema)
+	.or(directoryConfigSchema)
+	.or(rcloneConfigSchema);
+
+export const volumeConfigSchema = volumeConfigSchemaBase.onUndeclaredKey("delete");
 
 export type BackendConfig = typeof volumeConfigSchema.infer;
 

app/server/modules/lifecycle/startup.ts

@@ -10,6 +10,34 @@ import { VolumeHealthCheckJob } from "../../jobs/healthchecks";
 import { RepositoryHealthCheckJob } from "../../jobs/repository-healthchecks";
 import { BackupExecutionJob } from "../../jobs/backup-execution";
 import { CleanupSessionsJob } from "../../jobs/cleanup-sessions";
+import { repositoriesService } from "../repositories/repositories.service";
+import { notificationsService } from "../notifications/notifications.service";
+
+const ensureLatestConfigurationSchema = async () => {
+	const volumes = await db.query.volumesTable.findMany({});
+
+	for (const volume of volumes) {
+		await volumeService.updateVolume(volume.name, volume).catch((err) => {
+			logger.error(`Failed to update volume ${volume.name}: ${err}`);
+		});
+	}
+
+	const repositories = await db.query.repositoriesTable.findMany({});
+
+	for (const repo of repositories) {
+		await repositoriesService.updateRepository(repo.name, {}).catch((err) => {
+			logger.error(`Failed to update repository ${repo.name}: ${err}`);
+		});
+	}
+
+	const notifications = await db.query.notificationDestinationsTable.findMany({});
+
+	for (const notification of notifications) {
+		await notificationsService.updateDestination(notification.id, notification).catch((err) => {
+			logger.error(`Failed to update notification destination ${notification.id}: ${err}`);
+		});
+	}
+};
 
 export const startup = async () => {
 	await Scheduler.start();
@@ -19,6 +47,8 @@ export const startup = async () => {
 		logger.error(`Error ensuring restic passfile exists: ${err.message}`);
 	});
 
+	await ensureLatestConfigurationSchema();
+
 	const volumes = await db.query.volumesTable.findMany({
 		where: or(
 			eq(volumesTable.status, "mounted"),

app/server/modules/notifications/notifications.service.ts

@@ -11,8 +11,9 @@ import { cryptoUtils } from "../../utils/crypto";
 import { logger } from "../../utils/logger";
 import { sendNotification } from "../../utils/shoutrrr";
 import { buildShoutrrrUrl } from "./builders";
-import type { NotificationConfig, NotificationEvent } from "~/schemas/notifications";
+import { notificationConfigSchema, type NotificationConfig, type NotificationEvent } from "~/schemas/notifications";
 import { toMessage } from "../../utils/errors";
+import { type } from "arktype";
 
 const listDestinations = async () => {
 	const destinations = await db.query.notificationDestinationsTable.findMany({
@@ -187,12 +188,15 @@ const updateDestination = async (
 		updateData.enabled = updates.enabled;
 	}
 
-	if (updates.config !== undefined) {
+	const newConfig = notificationConfigSchema(updates.config || existing.config);
-		const encryptedConfig = await encryptSensitiveFields(updates.config);
+	if (newConfig instanceof type.errors) {
-		updateData.config = encryptedConfig;
+		throw new InternalServerError("Invalid notification configuration");
-		updateData.type = updates.config.type;
 	}
 
+	const encryptedConfig = await encryptSensitiveFields(newConfig);
+	updateData.config = encryptedConfig;
+	updateData.type = newConfig.type;
+
 	const [updated] = await db
 		.update(notificationDestinationsTable)
 		.set(updateData)

app/server/modules/repositories/repositories.service.ts

@@ -9,7 +9,13 @@ import { generateShortId } from "../../utils/id";
 import { restic } from "../../utils/restic";
 import { cryptoUtils } from "../../utils/crypto";
 import { repoMutex } from "../../core/repository-mutex";
-import type { CompressionMode, OverwriteMode, RepositoryConfig } from "~/schemas/restic";
+import {
+	repositoryConfigSchema,
+	type CompressionMode,
+	type OverwriteMode,
+	type RepositoryConfig,
+} from "~/schemas/restic";
+import { type } from "arktype";
 
 const listRepositories = async () => {
 	const repositories = await db.query.repositoriesTable.findMany({});
@@ -424,6 +430,13 @@ const updateRepository = async (name: string, updates: { name?: string; compress
 		throw new ConflictError("Cannot rename an imported local repository");
 	}
 
+	const newConfig = repositoryConfigSchema(existing.config);
+	if (newConfig instanceof type.errors) {
+		throw new InternalServerError("Invalid repository configuration");
+	}
+
+	const encryptedConfig = await encryptConfig(newConfig);
+
 	let newName = existing.name;
 	if (updates.name !== undefined && updates.name !== existing.name) {
 		const newSlug = slugify(updates.name, { lower: true, strict: true });
@@ -445,6 +458,7 @@ const updateRepository = async (name: string, updates: { name?: string; compress
 			name: newName,
 			compressionMode: updates.compressionMode ?? existing.compressionMode,
 			updatedAt: Date.now(),
+			config: encryptedConfig,
 		})
 		.where(eq(repositoriesTable.id, existing.id))
 		.returning();

app/server/modules/volumes/volume.service.ts

@@ -16,7 +16,8 @@ import type { UpdateVolumeBody } from "./volume.dto";
 import { getVolumePath } from "./helpers";
 import { logger } from "../../utils/logger";
 import { serverEvents } from "../../core/events";
-import type { BackendConfig } from "~/schemas/volumes";
+import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
+import { type } from "arktype";
 
 async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
 	switch (config.backend) {
@@ -192,7 +193,12 @@ const updateVolume = async (name: string, volumeData: UpdateVolumeBody) => {
 		await backend.unmount();
 	}
 
-	const encryptedConfig = volumeData.config ? await encryptSensitiveFields(volumeData.config) : undefined;
+	const newConfig = volumeConfigSchema(volumeData.config || existing.config);
+	if (newConfig instanceof type.errors) {
+		throw new InternalServerError("Invalid volume configuration");
+	}
+
+	const encryptedConfig = await encryptSensitiveFields(newConfig);
 
 	const [updated] = await db
 		.update(volumesTable)