From bba61c2a37b3901da77220febfa10d176f0612ad Mon Sep 17 00:00:00 2001 From: Nico <47644445+nicotsx@users.noreply.github.com> Date: Sat, 28 Feb 2026 00:02:00 +0100 Subject: [PATCH] fix: restic glob pattern in include (#594) Closes #590 ## Summary by CodeRabbit * **Tests** * Enhanced test coverage for backup pattern handling and anchoring * Added end-to-end scenarios validating include patterns, exclusion patterns, and exclude-if-present functionality * **Refactor** * Updated pattern processing logic to improve relative path resolution for backup patterns * Improved asynchronous handling in backup initialization workflow --- .../__tests__/backups.patterns.test.ts | 28 ++++- app/server/modules/backups/backup.helpers.ts | 10 +- app/server/utils/restic/commands/init.ts | 24 ++-- e2e/0002-backup-restore.spec.ts | 111 +++++++++++++++++- package.json | 2 +- playwright.config.ts | 1 - 6 files changed, 158 insertions(+), 18 deletions(-) diff --git a/app/server/modules/backups/__tests__/backups.patterns.test.ts b/app/server/modules/backups/__tests__/backups.patterns.test.ts index ada2bb20..0e9694c0 100644 --- a/app/server/modules/backups/__tests__/backups.patterns.test.ts +++ b/app/server/modules/backups/__tests__/backups.patterns.test.ts @@ -31,7 +31,12 @@ describe("executeBackup - include / exclude patterns", () => { // assert expect(options).toMatchObject({ - include: ["*.zip", path.join(volumePath, "Photos"), `!${path.join(volumePath, "Temp")}`, "!*.log"], + include: [ + path.join(volumePath, "*.zip"), + path.join(volumePath, "Photos"), + `!${path.join(volumePath, "Temp")}`, + `!${path.join(volumePath, "*.log")}`, + ], exclude: [".DS_Store", path.join(volumePath, "Config"), `!${path.join(volumePath, "Important")}`, "!*.tmp"], excludeIfPresent: [".nobackup"], }); @@ -68,7 +73,10 @@ describe("executeBackup - include / exclude patterns", () => { const options = createBackupOptions(schedule, volumePath, signal); // assert - expect(options.include).toEqual([relativeInclude, path.join(volumePath, "anchored/include")]); + expect(options.include).toEqual([ + path.join(volumePath, relativeInclude), + path.join(volumePath, "anchored/include"), + ]); }); test("should handle empty include and exclude patterns", () => { @@ -91,4 +99,20 @@ describe("executeBackup - include / exclude patterns", () => { expect(processPattern("relative/include", "/volume")).toBe("relative/include"); expect(processPattern("!*.log", "/volume")).toBe("!*.log"); }); + + test("anchors relative glob include patterns to the volume path", () => { + const volumePath = "/var/lib/zerobyte/volumes/vol123/_data"; + const schedule = createSchedule({ + includePatterns: ["**/*.xyz", "*.zip", "!**/*.tmp"], + }); + const signal = new AbortController().signal; + + const options = createBackupOptions(schedule, volumePath, signal); + + expect(options.include).toEqual([ + path.join(volumePath, "**/*.xyz"), + path.join(volumePath, "*.zip"), + `!${path.join(volumePath, "**/*.tmp")}`, + ]); + }); }); diff --git a/app/server/modules/backups/backup.helpers.ts b/app/server/modules/backups/backup.helpers.ts index ab5b3d7d..4a872d56 100644 --- a/app/server/modules/backups/backup.helpers.ts +++ b/app/server/modules/backups/backup.helpers.ts @@ -19,12 +19,14 @@ export const calculateNextRun = (cronExpression: string) => { } }; -export const processPattern = (pattern: string, volumePath: string) => { +export const processPattern = (pattern: string, volumePath: string, relative = false) => { const isNegated = pattern.startsWith("!"); const p = isNegated ? pattern.slice(1) : pattern; if (!p.startsWith("/")) { - return pattern; + if (!relative) return pattern; + const processed = path.join(volumePath, p); + return isNegated ? `!${processed}` : processed; } const processed = path.join(volumePath, p.slice(1)); @@ -37,5 +39,7 @@ export const createBackupOptions = (schedule: BackupSchedule, volumePath: string signal, exclude: schedule.excludePatterns ? schedule.excludePatterns.map((p) => processPattern(p, volumePath)) : undefined, excludeIfPresent: schedule.excludeIfPresent ?? undefined, - include: schedule.includePatterns ? schedule.includePatterns.map((p) => processPattern(p, volumePath)) : undefined, + include: schedule.includePatterns + ? schedule.includePatterns.map((p) => processPattern(p, volumePath, true)) + : undefined, }); diff --git a/app/server/utils/restic/commands/init.ts b/app/server/utils/restic/commands/init.ts index 0d1153d5..ea89683d 100644 --- a/app/server/utils/restic/commands/init.ts +++ b/app/server/utils/restic/commands/init.ts @@ -8,6 +8,19 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import { keyAdd } from "./key-add"; +const addDefaultKey = async (config: RepositoryConfig, organizationId: string, options?: { timeoutMs?: number }) => { + if (appConfig.resticHostname) { + const keyResult = await keyAdd(config, organizationId, { + host: appConfig.resticHostname, + timeoutMs: options?.timeoutMs, + }); + + if (!keyResult.success) { + logger.warn(`Repository initialized but failed to add key with hostname: ${keyResult.error}`); + } + } +}; + export const init = async (config: RepositoryConfig, organizationId: string, options?: { timeoutMs?: number }) => { const repoUrl = buildRepoUrl(config); @@ -28,16 +41,7 @@ export const init = async (config: RepositoryConfig, organizationId: string, opt logger.info(`Restic repository initialized: ${repoUrl}`); - if (appConfig.resticHostname) { - const keyResult = await keyAdd(config, organizationId, { - host: appConfig.resticHostname, - timeoutMs: options?.timeoutMs, - }); - - if (!keyResult.success) { - logger.warn(`Repository initialized but failed to add key with hostname: ${keyResult.error}`); - } - } + void addDefaultKey(config, organizationId, { timeoutMs: options?.timeoutMs }); return { success: true, error: null }; }; diff --git a/e2e/0002-backup-restore.spec.ts b/e2e/0002-backup-restore.spec.ts index 60ec9ca2..9062e12a 100644 --- a/e2e/0002-backup-restore.spec.ts +++ b/e2e/0002-backup-restore.spec.ts @@ -12,6 +12,12 @@ type ScenarioNames = { backupName: string; }; +type ScenarioOptions = { + includePatterns?: string; + excludePatterns?: string; + excludeIfPresent?: string; +}; + function getRunId(testInfo: TestInfo) { return `${testInfo.parallelIndex}-${testInfo.retry}-${randomUUID().slice(0, 8)}`; } @@ -34,7 +40,7 @@ function prepareTestFile(runId: string): string { return filePath; } -async function createBackupScenario(page: Page, names: ScenarioNames) { +async function createBackupScenario(page: Page, names: ScenarioNames, options: ScenarioOptions = {}) { await page.getByRole("button", { name: "Create Volume" }).click(); await page.getByRole("textbox", { name: "Name" }).fill(names.volumeName); await page.getByRole("button", { name: "test-data" }).click(); @@ -64,6 +70,15 @@ async function createBackupScenario(page: Page, names: ScenarioNames) { await page.getByRole("combobox").filter({ hasText: "Select frequency" }).click(); await page.getByRole("option", { name: "Daily" }).click(); await page.getByRole("textbox", { name: "Execution time" }).fill("00:00"); + if (options.includePatterns) { + await page.getByLabel("Additional include patterns").fill(options.includePatterns); + } + if (options.excludePatterns) { + await page.getByLabel("Exclusion patterns").fill(options.excludePatterns); + } + if (options.excludeIfPresent) { + await page.getByLabel("Exclude if file present").fill(options.excludeIfPresent); + } await page.getByRole("button", { name: "Create" }).click(); await expect(page.getByText("Backup job created successfully")).toBeVisible(); } @@ -127,3 +142,97 @@ test("deleting a volume cascades and removes its backup schedule", async ({ page await gotoAndWaitForAppReady(page, "/backups"); await expect(page.getByText(names.backupName, { exact: true })).toHaveCount(0); }); + +test("backup respects include globs, exclusion patterns, and exclude-if-present", async ({ page }, testInfo) => { + const runId = getRunId(testInfo); + const names = getScenarioNames(runId); + + const keptDir = `kept-${runId}`; + const secondKeptDir = `second-kept-${runId}`; + const blockedDir = `blocked-${runId}`; + const globOnlyDir = `glob-only-${runId}`; + const dataDir = `data-${runId}`; + const configDir = `config-${runId}`; + + const dataIncludedFile = `data-${runId}.txt`; + const configIncludedFile = `config-${runId}.json`; + const configExcludedFile = `secret-${runId}.json`; + const configNonJsonFile = `config-${runId}.txt`; + + const keptPath = path.join(testDataPath, keptDir); + const secondKeptPath = path.join(testDataPath, secondKeptDir); + const blockedPath = path.join(testDataPath, blockedDir); + const globOnlyPath = path.join(testDataPath, globOnlyDir); + const dataPath = path.join(testDataPath, dataDir); + const configPath = path.join(testDataPath, configDir); + + fs.mkdirSync(keptPath, { recursive: true }); + fs.mkdirSync(secondKeptPath, { recursive: true }); + fs.mkdirSync(blockedPath, { recursive: true }); + fs.mkdirSync(globOnlyPath, { recursive: true }); + fs.mkdirSync(dataPath, { recursive: true }); + fs.mkdirSync(configPath, { recursive: true }); + + fs.writeFileSync(path.join(keptPath, "keep.xyz"), "xyz content"); + fs.writeFileSync(path.join(keptPath, ".DS_Store"), "excluded metadata"); + fs.writeFileSync(path.join(keptPath, "skip.tmp"), "excluded tmp"); + + fs.writeFileSync(path.join(secondKeptPath, "second.xyz"), "xyz content"); + fs.writeFileSync(path.join(secondKeptPath, ".DS_Store"), "excluded metadata"); + + fs.writeFileSync(path.join(blockedPath, ".nobackup"), "marker"); + fs.writeFileSync(path.join(blockedPath, "blocked.xyz"), "should be excluded"); + + fs.writeFileSync(path.join(globOnlyPath, "glob-only.xyz"), "glob include"); + + fs.writeFileSync(path.join(dataPath, dataIncludedFile), "data include"); + fs.writeFileSync(path.join(configPath, configIncludedFile), "json include"); + fs.writeFileSync(path.join(configPath, configExcludedFile), "json excluded by absolute exclude"); + fs.writeFileSync(path.join(configPath, configNonJsonFile), "not included by /config/*.json"); + + await gotoAndWaitForAppReady(page, "/"); + await expect(page).toHaveURL("/volumes"); + + await createBackupScenario(page, names, { + includePatterns: [ + `/${keptDir}`, + `/${secondKeptDir}`, + `/${blockedDir}`, + `/${dataDir}/**`, + `/${configDir}/*.json`, + "**/*.xyz", + ].join("\n"), + excludePatterns: [".DS_Store", "*.tmp", `/${configDir}/secret*.json`].join("\n"), + excludeIfPresent: ".nobackup", + }); + + await page.getByRole("button", { name: "Backup now" }).click(); + await expect(page.getByText("Backup started successfully")).toBeVisible(); + await expect(page.getByText("✓ Success")).toBeVisible({ timeout: 30000 }); + + await page + .getByRole("button", { name: /\d+ B$/ }) + .first() + .click(); + await expect(page.getByText("File Browser")).toBeVisible(); + + for (const folder of [keptDir, secondKeptDir, globOnlyDir, dataDir, configDir, blockedDir]) { + const folderRow = page.getByRole("button", { name: folder, exact: true }); + await expect(folderRow).toBeVisible(); + await folderRow.locator("svg").first().click(); + } + + await expect(page.getByRole("button", { name: /keep\.xyz/ })).toBeVisible(); + await expect(page.getByRole("button", { name: /second\.xyz/ })).toBeVisible(); + await expect(page.getByRole("button", { name: /glob-only\.xyz/ })).toBeVisible(); + await expect(page.getByRole("button", { name: new RegExp(dataIncludedFile.replace(".", "\\.")) })).toBeVisible(); + await expect(page.getByRole("button", { name: new RegExp(configIncludedFile.replace(".", "\\.")) })).toBeVisible(); + + await expect(page.getByRole("button", { name: /\.DS_Store/ })).toHaveCount(0); + await expect(page.getByRole("button", { name: /skip\.tmp/ })).toHaveCount(0); + await expect(page.getByRole("button", { name: new RegExp(configExcludedFile.replace(".", "\\.")) })).toHaveCount(0); + await expect(page.getByRole("button", { name: new RegExp(configNonJsonFile.replace(".", "\\.")) })).toHaveCount(0); + + await expect(page.getByRole("button", { name: /\.nobackup/ })).toHaveCount(0); + await expect(page.getByRole("button", { name: /blocked\.xyz/ })).toHaveCount(0); +}); diff --git a/package.json b/package.json index 5811c29b..8f182aa3 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "tsc": "tsc --noEmit", "start:dev": "docker compose down && docker compose up --build zerobyte-dev", "start:prod": "docker compose down && docker compose up --build zerobyte-prod", - "start:e2e": "docker compose down && rm -f playwright/data/zerobyte.db playwright/data/cache.db playwright/data/restic.pass playwright/.auth/user.json playwright/restic.pass && docker compose up --build zerobyte-e2e", + "start:e2e": "docker compose down && rm -rf playwright/data/* playwright/.auth/user.json playwright/restic.pass playwright/temp && docker compose up --build zerobyte-e2e", "start:distroless": "docker compose down && docker compose up --build zerobyte-distroless", "gen:api-client": "dotenv -e .env.local -- openapi-ts", "gen:migrations": "dotenv -e .env.local -- drizzle-kit generate", diff --git a/playwright.config.ts b/playwright.config.ts index 4fe4fcf0..f0f98800 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,7 +5,6 @@ export default defineConfig({ testDir: "./e2e", fullyParallel: true, forbidOnly: !!process.env.CI, - timeout: 90000, retries: 0, reporter: "html", use: {