diff --git a/app/server/modules/notifications/notifications.service.ts b/app/server/modules/notifications/notifications.service.ts index 2d4d9b3a..ae553ebd 100644 --- a/app/server/modules/notifications/notifications.service.ts +++ b/app/server/modules/notifications/notifications.service.ts @@ -6,7 +6,7 @@ import { backupScheduleNotificationsTable, type NotificationDestination, } from "../../db/schema"; -import { logger } from "@zerobyte/core/node"; +import { logger, sanitizeSensitiveData } from "@zerobyte/core/node"; import { isAllowedWebhookUrl } from "@zerobyte/core/backup-hooks"; import { sendNotification } from "../../utils/shoutrrr"; import { formatDuration } from "~/utils/utils"; @@ -20,6 +20,12 @@ import { formatBytes } from "~/utils/format-bytes"; import { decryptNotificationConfig, encryptNotificationConfig } from "./notification-config-secrets"; import { serverEvents } from "~/server/core/events"; +const MAX_DELIVERY_ERROR_LENGTH = 2048; + +const formatDeliveryError = (error?: string) => { + return sanitizeSensitiveData(error ?? "Unknown error").slice(0, MAX_DELIVERY_ERROR_LENGTH); +}; + const getCustomShoutrrrWebhookUrl = (shoutrrrUrl: string) => { if (!URL.canParse(shoutrrrUrl)) { return null; @@ -195,7 +201,7 @@ const updateDeliveryStatus = async (destinationId: number, result: { success: bo .set({ status: result.success ? "healthy" : "error", lastChecked: Date.now(), - lastError: result.success ? null : (result.error ?? "Unknown error"), + lastError: result.success ? null : formatDeliveryError(result.error), updatedAt: Date.now(), }) .where(eq(notificationDestinationsTable.id, destinationId)) diff --git a/app/server/utils/shoutrrr.ts b/app/server/utils/shoutrrr.ts index 336ba7b6..e1ba525d 100644 --- a/app/server/utils/shoutrrr.ts +++ b/app/server/utils/shoutrrr.ts @@ -1,5 +1,4 @@ -import { safeExec } from "@zerobyte/core/node"; -import { logger } from "@zerobyte/core/node"; +import { logger, safeExec, sanitizeSensitiveData } from "@zerobyte/core/node"; import { toMessage } from "./errors"; interface SendNotificationParams { @@ -23,7 +22,7 @@ export async function sendNotification(params: SendNotificationParams) { return { success: true }; } - const errorMessage = result.stderr || result.stdout || "Unknown error"; + const errorMessage = sanitizeSensitiveData(result.stderr || result.stdout || "Unknown error"); logger.error(`Failed to send notification: ${errorMessage}`); return { success: false, diff --git a/e2e/0004-mirror-sync.spec.ts b/e2e/0004-mirror-sync.spec.ts index 382d2061..18caddba 100644 --- a/e2e/0004-mirror-sync.spec.ts +++ b/e2e/0004-mirror-sync.spec.ts @@ -12,10 +12,27 @@ type RepositorySummary = { shortId: string; }; +type MirrorAssignmentSummary = { + repositoryId: string; + lastCopyStatus: "success" | "error" | "in_progress" | null; +}; + +type MirrorSyncStatus = { + sourceCount: number; + mirrorCount: number; + missingSnapshots: unknown[]; +}; + function getRunId(testInfo: TestInfo) { return `${testInfo.parallelIndex}-${testInfo.retry}-${randomUUID().slice(0, 8)}`; } +function getBackupShortId(backupPageUrl: string) { + const backupShortId = new URL(backupPageUrl).pathname.split("/").filter(Boolean).at(-1); + expect(backupShortId).toBeTruthy(); + return backupShortId!; +} + function getWorkerTestDataPath() { fs.mkdirSync(testDataPath, { recursive: true }); return testDataPath; @@ -82,6 +99,25 @@ async function createBackupJob(page: Page, backupName: string, volumeName: strin await expect(page.getByText("Backup job created successfully")).toBeVisible(); } +async function waitForMirrorSyncComplete(page: Page, backupShortId: string, mirrorRepoShortId: string) { + await expect(async () => { + const mirrorsResponse = await page.request.get(`/api/v1/backups/${backupShortId}/mirrors`); + expect(mirrorsResponse.ok()).toBe(true); + const mirrors = (await mirrorsResponse.json()) as MirrorAssignmentSummary[]; + const mirror = mirrors.find((entry) => entry.repositoryId === mirrorRepoShortId); + expect(mirror?.lastCopyStatus).toBe("success"); + + const statusResponse = await page.request.get( + `/api/v1/backups/${backupShortId}/mirrors/${mirrorRepoShortId}/status`, + ); + expect(statusResponse.ok()).toBe(true); + const status = (await statusResponse.json()) as MirrorSyncStatus; + expect(status.sourceCount).toBe(1); + expect(status.mirrorCount).toBe(1); + expect(status.missingSnapshots).toHaveLength(0); + }).toPass({ timeout: 30000 }); +} + test("can sync missing snapshots to a mirror repository", async ({ page }, testInfo) => { const runId = getRunId(testInfo); const volumeName = `Volume-${runId}`; @@ -104,6 +140,7 @@ test("can sync missing snapshots to a mirror repository", async ({ page }, testI await createBackupJob(page, backupName, volumeName, primaryRepoName); const backupPageUrl = page.url(); + const backupShortId = getBackupShortId(backupPageUrl); const primaryRepoShortId = await getRepositoryShortId(page, primaryRepoName); const mirrorRepoShortId = await getRepositoryShortId(page, mirrorRepoName); @@ -142,20 +179,17 @@ test("can sync missing snapshots to a mirror repository", async ({ page }, testI const syncButton = syncDialog.getByRole("button", { name: "Sync 1 snapshots" }); await expect(syncButton).toBeEnabled(); - // Click sync button and wait for the mirror repository to contain the snapshot. + // Click sync and wait on the mirror status that backs this dialog and row. await syncButton.click(); - await gotoAndWaitForAppReady(page, `/repositories/${mirrorRepoShortId}`); - await page.getByRole("tab", { name: "Snapshots" }).click(); - await expect(page.getByText("Backup snapshots stored in this repository.")).toBeVisible(); - await expect(async () => { - await page.getByRole("button", { name: "Refresh" }).click(); - await expect(page.getByRole("checkbox", { name: /Select snapshot/ })).toHaveCount(1); - }).toPass({ timeout: 30000 }); + await waitForMirrorSyncComplete(page, backupShortId, mirrorRepoShortId); await gotoAndWaitForAppReady(page, backupPageUrl); // Open sync dialog again and verify all snapshots are synced - await mirrorRow.getByRole("button").first().click(); - await expect(syncDialog.getByRole("heading", { name: "Sync snapshots" })).toBeVisible(); + await expect(async () => { + await expect(mirrorRow).toBeVisible(); + await mirrorRow.getByRole("button").first().click(); + await expect(syncDialog.getByRole("heading", { name: "Sync snapshots" })).toBeVisible(); + }).toPass({ timeout: 15000 }); await expect(syncDialog.getByText(/All 1 snapshots are already synced/)).toBeVisible({ timeout: 15000 }); await syncDialog.getByRole("button", { name: "Cancel" }).click(); diff --git a/packages/core/src/backup-hooks/__tests__/hooks.test.ts b/packages/core/src/backup-hooks/__tests__/hooks.test.ts index 54b3870f..bbf65209 100644 --- a/packages/core/src/backup-hooks/__tests__/hooks.test.ts +++ b/packages/core/src/backup-hooks/__tests__/hooks.test.ts @@ -125,6 +125,33 @@ test("sends warning details to the post-backup webhook for a non-zero completed }); }); +test("sends warning details to the post-backup webhook for a zero-exit completed backup with warnings", async () => { + let postBody: WebhookBody | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as WebhookBody; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await runWithHooks({ + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + runBackup: () => completedBackup("snapshot-1", 0, "Backup was stopped by the user"), + }); + + expect(postBody).toMatchObject({ status: "warning", error: "Backup was stopped by the user" }); + expect(result).toEqual({ + status: "completed", + exitCode: 0, + result: "snapshot-1", + warningDetails: "Backup was stopped by the user", + }); +}); + test("sends error details to the post-backup webhook when the backup fails", async () => { let postBody: WebhookBody | undefined; diff --git a/packages/core/src/backup-hooks/index.ts b/packages/core/src/backup-hooks/index.ts index 4533ec47..4779e340 100644 --- a/packages/core/src/backup-hooks/index.ts +++ b/packages/core/src/backup-hooks/index.ts @@ -96,12 +96,16 @@ const appendDetails = (primary: string | null | undefined, next: string | null | return [primary, next].filter(Boolean).join("\n\n"); }; -const getCompletedStatus = (exitCode: number, signal: AbortSignal): BackupWebhookStatus => { +const getCompletedStatus = ( + exitCode: number, + warningDetails: string | null, + signal: AbortSignal, +): BackupWebhookStatus => { if (signal.aborted) { return "cancelled"; } - return exitCode === 0 ? "success" : "warning"; + return exitCode === 0 && !warningDetails ? "success" : "warning"; }; const createAbortController = (timeoutMs: number, signal?: AbortSignal) => { @@ -284,7 +288,7 @@ export const runBackupLifecycle = ({ Effect.map((result) => ({ status: "completed" as const, ...result, - hookStatus: getCompletedStatus(result.exitCode, signal), + hookStatus: getCompletedStatus(result.exitCode, result.warningDetails, signal), hookError: signal.aborted ? formatError(signal.reason) : (result.warningDetails ?? undefined), })), Effect.catchAll((error) => {