refactor(restic): split each command into its own file
This commit is contained in:
Nicolas Meienberger
Nico
parent
3113af8ce8
commit
ce7afafbf6
28 changed files with 2320 additions and 1564 deletions
|
|
@ -1,236 +0,0 @@
|
||||||
import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
|
|
||||||
import * as spawnModule from "./spawn";
|
|
||||||
import { buildRepoUrl, restic } from "./restic";
|
|
||||||
|
|
||||||
const successfulRestoreSummary = JSON.stringify({
|
|
||||||
message_type: "summary",
|
|
||||||
files_restored: 1,
|
|
||||||
files_skipped: 0,
|
|
||||||
bytes_skipped: 0,
|
|
||||||
});
|
|
||||||
|
|
||||||
let lastSafeSpawnArgs: string[] = [];
|
|
||||||
|
|
||||||
const safeSpawnMock = mock((params: spawnModule.SafeSpawnParams) => {
|
|
||||||
lastSafeSpawnArgs = params.args;
|
|
||||||
|
|
||||||
return Promise.resolve({
|
|
||||||
exitCode: 0,
|
|
||||||
summary: successfulRestoreSummary,
|
|
||||||
error: "",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const getRestoreArg = (args: string[]): string => {
|
|
||||||
const restoreIndex = args.indexOf("restore");
|
|
||||||
if (restoreIndex < 0) {
|
|
||||||
throw new Error("Expected restore command in restic arguments");
|
|
||||||
}
|
|
||||||
|
|
||||||
const restoreArg = args[restoreIndex + 1];
|
|
||||||
if (!restoreArg) {
|
|
||||||
throw new Error("Expected restore argument after restore command");
|
|
||||||
}
|
|
||||||
|
|
||||||
return restoreArg;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getOptionValues = (args: string[], option: string): string[] => {
|
|
||||||
const values: string[] = [];
|
|
||||||
for (let i = 0; i < args.length - 1; i++) {
|
|
||||||
if (args[i] === option) {
|
|
||||||
const value = args[i + 1];
|
|
||||||
if (value) {
|
|
||||||
values.push(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return values;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getLastSafeSpawnArgs = (): string[] => {
|
|
||||||
if (lastSafeSpawnArgs.length === 0) {
|
|
||||||
throw new Error("Expected safeSpawn to be called");
|
|
||||||
}
|
|
||||||
|
|
||||||
return lastSafeSpawnArgs;
|
|
||||||
};
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
safeSpawnMock.mockClear();
|
|
||||||
lastSafeSpawnArgs = [];
|
|
||||||
spyOn(spawnModule, "safeSpawn").mockImplementation(safeSpawnMock);
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
mock.restore();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("buildRepoUrl", () => {
|
|
||||||
describe("S3 backend", () => {
|
|
||||||
test("should build URL without trailing slash", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "s3" as const,
|
|
||||||
endpoint: "https://s3.amazonaws.com",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:https://s3.amazonaws.com/my-bucket");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should trim trailing slash from endpoint", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "s3" as const,
|
|
||||||
endpoint: "https://s3.xxxxxxxxx.net/",
|
|
||||||
bucket: "backup",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:https://s3.xxxxxxxxx.net/backup");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should trim trailing whitespace from endpoint", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "s3" as const,
|
|
||||||
endpoint: "https://s3.amazonaws.com/ ",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:https://s3.amazonaws.com/my-bucket");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should trim leading and trailing whitespace from endpoint", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "s3" as const,
|
|
||||||
endpoint: " https://s3.amazonaws.com/ ",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:https://s3.amazonaws.com/my-bucket");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("R2 backend", () => {
|
|
||||||
test("should build URL without trailing slash", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "r2" as const,
|
|
||||||
endpoint: "https://myaccount.r2.cloudflarestorage.com",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:myaccount.r2.cloudflarestorage.com/my-bucket");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should trim trailing slash from endpoint", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "r2" as const,
|
|
||||||
endpoint: "https://myaccount.r2.cloudflarestorage.com/",
|
|
||||||
bucket: "backup",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:myaccount.r2.cloudflarestorage.com/backup");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should strip protocol and trailing slash", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "r2" as const,
|
|
||||||
endpoint: "https://myaccount.r2.cloudflarestorage.com/",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:myaccount.r2.cloudflarestorage.com/my-bucket");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("should trim whitespace and strip protocol", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "r2" as const,
|
|
||||||
endpoint: " https://myaccount.r2.cloudflarestorage.com/ ",
|
|
||||||
bucket: "my-bucket",
|
|
||||||
accessKeyId: "test",
|
|
||||||
secretAccessKey: "test",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("s3:myaccount.r2.cloudflarestorage.com/my-bucket");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("other backends", () => {
|
|
||||||
test("should build local repository URL", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "local" as const,
|
|
||||||
path: "/path/to/repo",
|
|
||||||
};
|
|
||||||
expect(buildRepoUrl(config)).toBe("/path/to/repo");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("restore", () => {
|
|
||||||
const config = {
|
|
||||||
backend: "local" as const,
|
|
||||||
path: "/tmp/restic-repo",
|
|
||||||
isExistingRepository: true,
|
|
||||||
customPassword: "custom-password",
|
|
||||||
};
|
|
||||||
|
|
||||||
test("keeps snapshot restore arg and absolute include paths when target is root", async () => {
|
|
||||||
await restic.restore(config, "snapshot-123", "/", {
|
|
||||||
organizationId: "org-1",
|
|
||||||
include: [
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const args = getLastSafeSpawnArgs();
|
|
||||||
expect(getRestoreArg(args)).toBe("snapshot-123");
|
|
||||||
expect(getOptionValues(args, "--include")).toEqual([
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("restores from common ancestor and strips include paths for non-root targets", async () => {
|
|
||||||
await restic.restore(config, "snapshot-456", "/tmp/restore-target", {
|
|
||||||
organizationId: "org-1",
|
|
||||||
include: [
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
|
||||||
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const args = getLastSafeSpawnArgs();
|
|
||||||
expect(getRestoreArg(args)).toBe("snapshot-456:/var/lib/zerobyte/volumes/vol123/_data");
|
|
||||||
expect(getOptionValues(args, "--include")).toEqual(["Documents/report.pdf", "Photos/summer.jpg"]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("uses base path for non-root restore when includes are omitted", async () => {
|
|
||||||
await restic.restore(config, "snapshot-789", "/tmp/restore-target", {
|
|
||||||
organizationId: "org-1",
|
|
||||||
basePath: "/var/lib/zerobyte/volumes/vol123/_data",
|
|
||||||
});
|
|
||||||
|
|
||||||
const args = getLastSafeSpawnArgs();
|
|
||||||
expect(getRestoreArg(args)).toBe("snapshot-789:/var/lib/zerobyte/volumes/vol123/_data");
|
|
||||||
expect(getOptionValues(args, "--include")).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("does not pass an empty include when include equals restore root", async () => {
|
|
||||||
await restic.restore(config, "snapshot-7202d8cc", "/Users/nicolas/Documents/restore", {
|
|
||||||
organizationId: "org-1",
|
|
||||||
include: ["/Users/nicolas/Developer/zerobyte/tmp/deep/test/files"],
|
|
||||||
overwrite: "always",
|
|
||||||
});
|
|
||||||
|
|
||||||
const args = getLastSafeSpawnArgs();
|
|
||||||
expect(getRestoreArg(args)).toBe("snapshot-7202d8cc:/Users/nicolas/Developer/zerobyte/tmp/deep/test/files");
|
|
||||||
expect(getOptionValues(args, "--include")).toEqual([]);
|
|
||||||
expect(args).not.toContain("");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
File diff suppressed because it is too large
Load diff
324
app/server/utils/restic/commands/__tests__/backup.test.ts
Normal file
324
app/server/utils/restic/commands/__tests__/backup.test.ts
Normal file
|
|
@ -0,0 +1,324 @@
|
||||||
|
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test";
|
||||||
|
import * as cleanupModule from "~/server/utils/restic/helpers/cleanup-temporary-keys";
|
||||||
|
import * as spawnModule from "~/server/utils/spawn";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { backup } from "../backup";
|
||||||
|
|
||||||
|
const VALID_SUMMARY = JSON.stringify({
|
||||||
|
message_type: "summary",
|
||||||
|
files_new: 10,
|
||||||
|
files_changed: 5,
|
||||||
|
files_unmodified: 85,
|
||||||
|
dirs_new: 2,
|
||||||
|
dirs_changed: 1,
|
||||||
|
dirs_unmodified: 17,
|
||||||
|
data_blobs: 20,
|
||||||
|
tree_blobs: 5,
|
||||||
|
data_added: 1048576,
|
||||||
|
total_files_processed: 100,
|
||||||
|
total_bytes_processed: 2097152,
|
||||||
|
total_duration: 12.34,
|
||||||
|
snapshot_id: "abcd1234",
|
||||||
|
});
|
||||||
|
|
||||||
|
const VALID_PROGRESS_LINE = JSON.stringify({
|
||||||
|
message_type: "status",
|
||||||
|
seconds_elapsed: 5,
|
||||||
|
percent_done: 0.5,
|
||||||
|
total_files: 100,
|
||||||
|
files_done: 50,
|
||||||
|
total_bytes: 2097152,
|
||||||
|
bytes_done: 1048576,
|
||||||
|
});
|
||||||
|
|
||||||
|
const config = {
|
||||||
|
backend: "local" as const,
|
||||||
|
path: "/tmp/restic-repo",
|
||||||
|
isExistingRepository: true,
|
||||||
|
customPassword: "custom-password",
|
||||||
|
};
|
||||||
|
|
||||||
|
type SetupOptions = {
|
||||||
|
spawnResult?: Partial<spawnModule.SpawnResult>;
|
||||||
|
onSpawnCall?: (params: spawnModule.SafeSpawnParams) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets up mocks for safeSpawn and cleanupTemporaryKeys, captures spawn args,
|
||||||
|
* and returns helpers to inspect what was passed to restic.
|
||||||
|
*/
|
||||||
|
const setup = ({ spawnResult = {}, onSpawnCall }: SetupOptions = {}) => {
|
||||||
|
let capturedArgs: string[] = [];
|
||||||
|
|
||||||
|
const cleanupSpy = spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
|
||||||
|
spyOn(spawnModule, "safeSpawn").mockImplementation((params) => {
|
||||||
|
capturedArgs = params.args;
|
||||||
|
onSpawnCall?.(params);
|
||||||
|
return Promise.resolve({ exitCode: 0, summary: VALID_SUMMARY, error: "", ...spawnResult });
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
cleanupSpy,
|
||||||
|
getArgs: () => capturedArgs,
|
||||||
|
hasFlag: (flag: string) => capturedArgs.includes(flag),
|
||||||
|
getOptionValues: (option: string): string[] => {
|
||||||
|
const values: string[] = [];
|
||||||
|
for (let i = 0; i < capturedArgs.length - 1; i++) {
|
||||||
|
if (capturedArgs[i] === option && capturedArgs[i + 1]) {
|
||||||
|
values.push(capturedArgs[i + 1]!);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
mock.restore();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("backup command", () => {
|
||||||
|
describe("argument construction", () => {
|
||||||
|
test("passes source path as positional arg when no include list is given", async () => {
|
||||||
|
const { getArgs, hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(getArgs()).toContain("/mnt/data");
|
||||||
|
expect(hasFlag("--files-from")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses --files-from instead of source path when include list is provided", async () => {
|
||||||
|
const { hasFlag, getArgs } = setup();
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
include: ["/mnt/data/docs", "/mnt/data/photos"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(hasFlag("--files-from")).toBe(true);
|
||||||
|
expect(getArgs()).not.toContain("/mnt/data");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("adds --tag for each entry in options.tags", async () => {
|
||||||
|
const { getOptionValues } = setup();
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
tags: ["tag-a", "tag-b"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getOptionValues("--tag")).toEqual(["tag-a", "tag-b"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits --tag when tags list is empty", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1", tags: [] });
|
||||||
|
|
||||||
|
expect(hasFlag("--tag")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("passes provided compressionMode to --compression", async () => {
|
||||||
|
const { getOptionValues } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1", compressionMode: "max" });
|
||||||
|
|
||||||
|
expect(getOptionValues("--compression")).toEqual(["max"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("defaults --compression to auto when compressionMode is omitted", async () => {
|
||||||
|
const { getOptionValues } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(getOptionValues("--compression")).toEqual(["auto"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("adds --one-file-system when oneFileSystem is true", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1", oneFileSystem: true });
|
||||||
|
|
||||||
|
expect(hasFlag("--one-file-system")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits --one-file-system when oneFileSystem is false", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1", oneFileSystem: false });
|
||||||
|
|
||||||
|
expect(hasFlag("--one-file-system")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("adds --exclude-file when exclude list is provided", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
exclude: ["/mnt/data/tmp", "/mnt/data/cache"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(hasFlag("--exclude-file")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits --exclude-file when exclude list is empty", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1", exclude: [] });
|
||||||
|
|
||||||
|
expect(hasFlag("--exclude-file")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("adds --exclude-if-present for each entry in excludeIfPresent", async () => {
|
||||||
|
const { getOptionValues } = setup();
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
excludeIfPresent: [".nobackup", ".gitignore"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getOptionValues("--exclude-if-present")).toEqual([".nobackup", ".gitignore"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("always includes DEFAULT_EXCLUDES as --exclude args", async () => {
|
||||||
|
const { getOptionValues } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(getOptionValues("--exclude").length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("includes --host arg from config", async () => {
|
||||||
|
const { hasFlag } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(hasFlag("--host")).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("exit code handling", () => {
|
||||||
|
test("returns parsed result on exit code 0", async () => {
|
||||||
|
setup();
|
||||||
|
const { result, exitCode } = await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(exitCode).toBe(0);
|
||||||
|
expect(result?.snapshot_id).toBe("abcd1234");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns result without throwing on exit code 3 (partial read errors)", async () => {
|
||||||
|
setup({ spawnResult: { exitCode: 3 } });
|
||||||
|
const { result, exitCode } = await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(exitCode).toBe(3);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("throws ResticError on non-zero, non-3 exit codes", async () => {
|
||||||
|
setup({ spawnResult: { exitCode: 1, summary: "", error: "fatal error" } });
|
||||||
|
|
||||||
|
await expect(backup(config, "/mnt/data", { organizationId: "org-1" })).rejects.toBeInstanceOf(ResticError);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("preserves the exit code inside the thrown ResticError", async () => {
|
||||||
|
setup({ spawnResult: { exitCode: 12, summary: "", error: "wrong password" } });
|
||||||
|
|
||||||
|
const error = await backup(config, "/mnt/data", { organizationId: "org-1" }).catch((e) => e);
|
||||||
|
expect(error).toBeInstanceOf(ResticError);
|
||||||
|
expect((error as ResticError).code).toBe(12);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns { result: null } when the abort signal is triggered", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
setup({
|
||||||
|
onSpawnCall: () => controller.abort(),
|
||||||
|
spawnResult: { exitCode: 130, summary: "", error: "" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const { result, exitCode } = await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(exitCode).toBe(130);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("output parsing", () => {
|
||||||
|
test("returns a fully parsed summary object on valid output", async () => {
|
||||||
|
setup();
|
||||||
|
const { result } = await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
message_type: "summary",
|
||||||
|
snapshot_id: "abcd1234",
|
||||||
|
total_duration: 12.34,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns { result: null } when summary line is not valid JSON", async () => {
|
||||||
|
setup({ spawnResult: { summary: "not-json" } });
|
||||||
|
const { result } = await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns { result: null } when summary JSON does not satisfy the schema", async () => {
|
||||||
|
setup({ spawnResult: { summary: JSON.stringify({ message_type: "summary" }) } });
|
||||||
|
const { result } = await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("progress callbacks", () => {
|
||||||
|
test("calls onProgress with parsed data when a valid status line arrives", async () => {
|
||||||
|
const progressUpdates: unknown[] = [];
|
||||||
|
setup({ onSpawnCall: (params) => params.onStdout?.(VALID_PROGRESS_LINE) });
|
||||||
|
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
onProgress: (p) => progressUpdates.push(p),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(progressUpdates.length).toBeGreaterThan(0);
|
||||||
|
expect(progressUpdates[0]).toMatchObject({
|
||||||
|
message_type: "status",
|
||||||
|
percent_done: 0.5,
|
||||||
|
files_done: 50,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ignores non-JSON stdout lines without throwing", async () => {
|
||||||
|
setup({
|
||||||
|
onSpawnCall: (params) => {
|
||||||
|
params.onStdout?.("scanning...");
|
||||||
|
params.onStdout?.("repository opened");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
backup(config, "/mnt/data", { organizationId: "org-1", onProgress: () => {} }),
|
||||||
|
).resolves.toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ignores valid JSON lines that do not match the progress schema", async () => {
|
||||||
|
const progressUpdates: unknown[] = [];
|
||||||
|
setup({
|
||||||
|
onSpawnCall: (params) => params.onStdout?.(JSON.stringify({ message_type: "verbose_status", action: "scan" })),
|
||||||
|
});
|
||||||
|
|
||||||
|
await backup(config, "/mnt/data", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
onProgress: (p) => progressUpdates.push(p),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(progressUpdates).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("cleanup", () => {
|
||||||
|
test("calls cleanupTemporaryKeys after a successful backup", async () => {
|
||||||
|
const { cleanupSpy } = setup();
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" });
|
||||||
|
|
||||||
|
expect(cleanupSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("calls cleanupTemporaryKeys even when the command fails", async () => {
|
||||||
|
const { cleanupSpy } = setup({ spawnResult: { exitCode: 1, summary: "", error: "fail" } });
|
||||||
|
await backup(config, "/mnt/data", { organizationId: "org-1" }).catch(() => {});
|
||||||
|
|
||||||
|
expect(cleanupSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
115
app/server/utils/restic/commands/__tests__/restore.test.ts
Normal file
115
app/server/utils/restic/commands/__tests__/restore.test.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test";
|
||||||
|
import * as spawnModule from "~/server/utils/spawn";
|
||||||
|
import { restore } from "../restore";
|
||||||
|
|
||||||
|
const successfulRestoreSummary = JSON.stringify({
|
||||||
|
message_type: "summary",
|
||||||
|
files_restored: 1,
|
||||||
|
files_skipped: 0,
|
||||||
|
bytes_skipped: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const config = {
|
||||||
|
backend: "local" as const,
|
||||||
|
path: "/tmp/restic-repo",
|
||||||
|
isExistingRepository: true,
|
||||||
|
customPassword: "custom-password",
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets up the safeSpawn mock and returns helpers to inspect the restic args
|
||||||
|
* that were built for the restore command.
|
||||||
|
*/
|
||||||
|
const setup = () => {
|
||||||
|
let capturedArgs: string[] = [];
|
||||||
|
|
||||||
|
spyOn(spawnModule, "safeSpawn").mockImplementation((params) => {
|
||||||
|
capturedArgs = params.args;
|
||||||
|
return Promise.resolve({ exitCode: 0, summary: successfulRestoreSummary, error: "" });
|
||||||
|
});
|
||||||
|
|
||||||
|
const getRestoreArg = () => {
|
||||||
|
const restoreIndex = capturedArgs.indexOf("restore");
|
||||||
|
if (restoreIndex < 0 || !capturedArgs[restoreIndex + 1]) {
|
||||||
|
throw new Error("Expected restore argument after restore command");
|
||||||
|
}
|
||||||
|
return capturedArgs[restoreIndex + 1]!;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getOptionValues = (option: string): string[] => {
|
||||||
|
const values: string[] = [];
|
||||||
|
for (let i = 0; i < capturedArgs.length - 1; i++) {
|
||||||
|
if (capturedArgs[i] === option && capturedArgs[i + 1]) {
|
||||||
|
values.push(capturedArgs[i + 1]!);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
getArgs: () => capturedArgs,
|
||||||
|
getRestoreArg,
|
||||||
|
getOptionValues,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
mock.restore();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("restore command", () => {
|
||||||
|
test("keeps snapshot restore arg and absolute include paths when target is root", async () => {
|
||||||
|
const { getRestoreArg, getOptionValues } = setup();
|
||||||
|
await restore(config, "snapshot-123", "/", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
include: [
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getRestoreArg()).toBe("snapshot-123");
|
||||||
|
expect(getOptionValues("--include")).toEqual([
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("restores from common ancestor and strips include paths for non-root targets", async () => {
|
||||||
|
const { getRestoreArg, getOptionValues } = setup();
|
||||||
|
await restore(config, "snapshot-456", "/tmp/restore-target", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
include: [
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Documents/report.pdf",
|
||||||
|
"/var/lib/zerobyte/volumes/vol123/_data/Photos/summer.jpg",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getRestoreArg()).toBe("snapshot-456:/var/lib/zerobyte/volumes/vol123/_data");
|
||||||
|
expect(getOptionValues("--include")).toEqual(["Documents/report.pdf", "Photos/summer.jpg"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses base path for non-root restore when includes are omitted", async () => {
|
||||||
|
const { getRestoreArg, getOptionValues } = setup();
|
||||||
|
await restore(config, "snapshot-789", "/tmp/restore-target", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
basePath: "/var/lib/zerobyte/volumes/vol123/_data",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getRestoreArg()).toBe("snapshot-789:/var/lib/zerobyte/volumes/vol123/_data");
|
||||||
|
expect(getOptionValues("--include")).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not pass an empty include when include equals restore root", async () => {
|
||||||
|
const { getArgs, getRestoreArg, getOptionValues } = setup();
|
||||||
|
await restore(config, "snapshot-7202d8cc", "/Users/nicolas/Documents/restore", {
|
||||||
|
organizationId: "org-1",
|
||||||
|
include: ["/Users/nicolas/Developer/zerobyte/tmp/deep/test/files"],
|
||||||
|
overwrite: "always",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getRestoreArg()).toBe("snapshot-7202d8cc:/Users/nicolas/Developer/zerobyte/tmp/deep/test/files");
|
||||||
|
expect(getOptionValues("--include")).toEqual([]);
|
||||||
|
expect(getArgs()).not.toContain("");
|
||||||
|
});
|
||||||
|
});
|
||||||
166
app/server/utils/restic/commands/backup.ts
Normal file
166
app/server/utils/restic/commands/backup.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { type } from "arktype";
|
||||||
|
import { throttle } from "es-toolkit";
|
||||||
|
import type { CompressionMode, RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import {
|
||||||
|
type ResticBackupProgressDto,
|
||||||
|
resticBackupOutputSchema,
|
||||||
|
resticBackupProgressSchema,
|
||||||
|
} from "~/schemas/restic-dto";
|
||||||
|
import { DEFAULT_EXCLUDES } from "~/server/core/constants";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeSpawn } from "~/server/utils/spawn";
|
||||||
|
import { config as appConfig } from "~/server/core/config";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const backup = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
source: string,
|
||||||
|
options: {
|
||||||
|
organizationId: string;
|
||||||
|
exclude?: string[];
|
||||||
|
excludeIfPresent?: string[];
|
||||||
|
include?: string[];
|
||||||
|
tags?: string[];
|
||||||
|
oneFileSystem?: boolean;
|
||||||
|
compressionMode?: CompressionMode;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
onProgress?: (progress: ResticBackupProgressDto) => void;
|
||||||
|
},
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "backup", "--compression", options.compressionMode ?? "auto"];
|
||||||
|
|
||||||
|
if (options.oneFileSystem) {
|
||||||
|
args.push("--one-file-system");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (appConfig.resticHostname) {
|
||||||
|
args.push("--host", appConfig.resticHostname);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.tags && options.tags.length > 0) {
|
||||||
|
for (const tag of options.tags) {
|
||||||
|
args.push("--tag", tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let includeFile: string | null = null;
|
||||||
|
if (options.include && options.include.length > 0) {
|
||||||
|
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-restic-include-"));
|
||||||
|
includeFile = path.join(tmp, "include.txt");
|
||||||
|
|
||||||
|
await fs.writeFile(includeFile, options.include.join("\n"), "utf-8");
|
||||||
|
|
||||||
|
args.push("--files-from", includeFile);
|
||||||
|
} else {
|
||||||
|
args.push(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const exclude of DEFAULT_EXCLUDES) {
|
||||||
|
args.push("--exclude", exclude);
|
||||||
|
}
|
||||||
|
|
||||||
|
let excludeFile: string | null = null;
|
||||||
|
if (options.exclude && options.exclude.length > 0) {
|
||||||
|
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-restic-exclude-"));
|
||||||
|
excludeFile = path.join(tmp, "exclude.txt");
|
||||||
|
|
||||||
|
await fs.writeFile(excludeFile, options.exclude.join("\n"), "utf-8");
|
||||||
|
|
||||||
|
args.push("--exclude-file", excludeFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.excludeIfPresent && options.excludeIfPresent.length > 0) {
|
||||||
|
for (const filename of options.excludeIfPresent) {
|
||||||
|
args.push("--exclude-if-present", filename);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const logData = throttle((data: string) => {
|
||||||
|
logger.info(data.trim());
|
||||||
|
}, 5000);
|
||||||
|
|
||||||
|
const streamProgress = throttle((data: string) => {
|
||||||
|
if (options.onProgress) {
|
||||||
|
try {
|
||||||
|
const jsonData = JSON.parse(data);
|
||||||
|
const progress = resticBackupProgressSchema(jsonData);
|
||||||
|
if (!(progress instanceof type.errors)) {
|
||||||
|
options.onProgress(progress);
|
||||||
|
} else {
|
||||||
|
logger.error(progress.summary);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore JSON parse errors for non-JSON lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 1000);
|
||||||
|
|
||||||
|
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||||
|
const res = await safeSpawn({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: options.signal,
|
||||||
|
onStdout: (data) => {
|
||||||
|
logData(data);
|
||||||
|
if (options.onProgress) {
|
||||||
|
streamProgress(data);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (includeFile) {
|
||||||
|
await fs.unlink(includeFile).catch(() => {});
|
||||||
|
}
|
||||||
|
if (excludeFile) {
|
||||||
|
await fs.unlink(excludeFile).catch(() => {});
|
||||||
|
}
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (options.signal?.aborted) {
|
||||||
|
logger.warn("Restic backup was aborted by signal.");
|
||||||
|
return { result: null, exitCode: res.exitCode };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (res.exitCode === 3) {
|
||||||
|
logger.error(`Restic backup encountered read errors: ${res.error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (res.exitCode !== 0 && res.exitCode !== 3) {
|
||||||
|
logger.error(`Restic backup failed: ${res.error}`);
|
||||||
|
logger.error(`Command executed: restic ${args.join(" ")}`);
|
||||||
|
|
||||||
|
throw new ResticError(res.exitCode, res.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastLine = res.summary.trim();
|
||||||
|
let summaryLine: unknown = {};
|
||||||
|
try {
|
||||||
|
summaryLine = JSON.parse(lastLine ?? "{}");
|
||||||
|
} catch {
|
||||||
|
logger.warn("Failed to parse restic backup output JSON summary.", lastLine);
|
||||||
|
summaryLine = {};
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`Restic backup output last line: ${JSON.stringify(summaryLine)}`);
|
||||||
|
const result = resticBackupOutputSchema(summaryLine);
|
||||||
|
|
||||||
|
if (result instanceof type.errors) {
|
||||||
|
logger.error(`Restic backup output validation failed: ${result.summary}`);
|
||||||
|
return { result: null, exitCode: res.exitCode };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { result, exitCode: res.exitCode };
|
||||||
|
};
|
||||||
67
app/server/utils/restic/commands/check.ts
Normal file
67
app/server/utils/restic/commands/check.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const check = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
options: {
|
||||||
|
readData?: boolean;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
organizationId: string;
|
||||||
|
},
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "check"];
|
||||||
|
|
||||||
|
if (options.readData) {
|
||||||
|
args.push("--read-data");
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (options.signal?.aborted) {
|
||||||
|
logger.warn("Restic check was aborted by signal.");
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
hasErrors: true,
|
||||||
|
output: "",
|
||||||
|
error: "Operation aborted",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const { stdout, stderr } = res;
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic check failed: ${stderr}`);
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
hasErrors: true,
|
||||||
|
output: stdout,
|
||||||
|
error: stderr,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasErrors = stdout.includes("Fatal");
|
||||||
|
|
||||||
|
logger.info(`Restic check completed for repository: ${repoUrl}`);
|
||||||
|
return {
|
||||||
|
success: !hasErrors,
|
||||||
|
hasErrors,
|
||||||
|
output: stdout,
|
||||||
|
error: hasErrors ? "Repository contains errors" : null,
|
||||||
|
};
|
||||||
|
};
|
||||||
73
app/server/utils/restic/commands/copy.ts
Normal file
73
app/server/utils/restic/commands/copy.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { formatBandwidthLimit } from "../helpers/bandwidth";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const copy = async (
|
||||||
|
sourceConfig: RepositoryConfig,
|
||||||
|
destConfig: RepositoryConfig,
|
||||||
|
options: { organizationId: string; tag?: string; snapshotId?: string },
|
||||||
|
) => {
|
||||||
|
const sourceRepoUrl = buildRepoUrl(sourceConfig);
|
||||||
|
const destRepoUrl = buildRepoUrl(destConfig);
|
||||||
|
|
||||||
|
const sourceEnv = await buildEnv(sourceConfig, options.organizationId);
|
||||||
|
const destEnv = await buildEnv(destConfig, options.organizationId);
|
||||||
|
|
||||||
|
const env: Record<string, string> = {
|
||||||
|
...sourceEnv,
|
||||||
|
...destEnv,
|
||||||
|
RESTIC_FROM_PASSWORD_FILE: sourceEnv.RESTIC_PASSWORD_FILE,
|
||||||
|
};
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", destRepoUrl, "copy", "--from-repo", sourceRepoUrl];
|
||||||
|
|
||||||
|
if (options.tag) {
|
||||||
|
args.push("--tag", options.tag);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.snapshotId) {
|
||||||
|
args.push(options.snapshotId);
|
||||||
|
} else {
|
||||||
|
args.push("latest");
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, destConfig, { skipBandwidth: true });
|
||||||
|
|
||||||
|
const sourceDownloadLimit = formatBandwidthLimit(sourceConfig.downloadLimit);
|
||||||
|
const destUploadLimit = formatBandwidthLimit(destConfig.uploadLimit);
|
||||||
|
|
||||||
|
if (sourceDownloadLimit) {
|
||||||
|
args.push("--limit-download", sourceDownloadLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (destUploadLimit) {
|
||||||
|
args.push("--limit-upload", destUploadLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Copying snapshots from ${sourceRepoUrl} to ${destRepoUrl}...`);
|
||||||
|
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
|
||||||
|
await cleanupTemporaryKeys(sourceEnv);
|
||||||
|
await cleanupTemporaryKeys(destEnv);
|
||||||
|
|
||||||
|
const { stdout, stderr } = res;
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic copy failed: ${stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Restic copy completed from ${sourceRepoUrl} to ${destRepoUrl}`);
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
output: stdout,
|
||||||
|
};
|
||||||
|
};
|
||||||
34
app/server/utils/restic/commands/delete-snapshots.ts
Normal file
34
app/server/utils/restic/commands/delete-snapshots.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[], organizationId: string) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
if (snapshotIds.length === 0) {
|
||||||
|
throw new Error("No snapshot IDs provided for deletion.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "forget", ...snapshotIds, "--prune"];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic snapshot deletion failed: ${res.stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, res.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true };
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deleteSnapshot = async (config: RepositoryConfig, snapshotId: string, organizationId: string) => {
|
||||||
|
return deleteSnapshots(config, [snapshotId], organizationId);
|
||||||
|
};
|
||||||
111
app/server/utils/restic/commands/dump.ts
Normal file
111
app/server/utils/restic/commands/dump.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { normalizeAbsolutePath } from "~/utils/path";
|
||||||
|
import type { Readable } from "node:stream";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { safeSpawn } from "~/server/utils/spawn";
|
||||||
|
import type { ResticDumpStream } from "../types";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
const normalizeDumpPath = (pathToDump?: string): string => {
|
||||||
|
const trimmedPath = pathToDump?.trim();
|
||||||
|
if (!trimmedPath) {
|
||||||
|
return "/";
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalizeAbsolutePath(trimmedPath);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const dump = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
snapshotRef: string,
|
||||||
|
options: {
|
||||||
|
organizationId: string;
|
||||||
|
path?: string;
|
||||||
|
archive?: false;
|
||||||
|
},
|
||||||
|
): Promise<ResticDumpStream> => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
const pathToDump = normalizeDumpPath(options.path);
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "dump", snapshotRef, pathToDump];
|
||||||
|
|
||||||
|
if (options.archive !== false) {
|
||||||
|
args.push("--archive", "tar");
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config, { includeJson: false });
|
||||||
|
|
||||||
|
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||||
|
|
||||||
|
let didCleanup = false;
|
||||||
|
const cleanup = async () => {
|
||||||
|
if (didCleanup) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
didCleanup = true;
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
};
|
||||||
|
|
||||||
|
let stream: Readable | null = null;
|
||||||
|
let abortController: AbortController | null = new AbortController();
|
||||||
|
|
||||||
|
const maxStderrChars = 64 * 1024;
|
||||||
|
let stderrTail = "";
|
||||||
|
|
||||||
|
const completion = safeSpawn({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: abortController.signal,
|
||||||
|
stdoutMode: "raw",
|
||||||
|
onSpawn: (child) => {
|
||||||
|
stream = child.stdout;
|
||||||
|
},
|
||||||
|
onStderr: (line) => {
|
||||||
|
const chunk = line.trim();
|
||||||
|
if (chunk) {
|
||||||
|
stderrTail += `${line}\n`;
|
||||||
|
if (stderrTail.length > maxStderrChars) {
|
||||||
|
stderrTail = stderrTail.slice(-maxStderrChars);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.then((result) => {
|
||||||
|
if (result.exitCode === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stderr = stderrTail.trim() || result.error;
|
||||||
|
logger.error(`Restic dump failed: ${stderr}`);
|
||||||
|
throw new ResticError(result.exitCode, stderr);
|
||||||
|
})
|
||||||
|
.finally(async () => {
|
||||||
|
abortController = null;
|
||||||
|
await cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
completion.catch(() => {});
|
||||||
|
const completionPromise = new Promise<void>((resolve, reject) => completion.then(resolve, reject));
|
||||||
|
|
||||||
|
if (!stream) {
|
||||||
|
await cleanup();
|
||||||
|
throw new Error("Failed to initialize restic dump stream");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
stream,
|
||||||
|
completion: completionPromise,
|
||||||
|
abort: () => {
|
||||||
|
if (abortController) {
|
||||||
|
abortController.abort();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
67
app/server/utils/restic/commands/forget.ts
Normal file
67
app/server/utils/restic/commands/forget.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import type { RetentionPolicy } from "~/server/modules/backups/backups.dto";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { safeJsonParse } from "~/server/utils/json";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import type { ResticForgetResponse } from "../types";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const forget = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
options: RetentionPolicy,
|
||||||
|
extra: { tag: string; organizationId: string; dryRun?: boolean },
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, extra.organizationId);
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag];
|
||||||
|
|
||||||
|
if (extra.dryRun) {
|
||||||
|
args.push("--dry-run", "--no-lock");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.keepLast) {
|
||||||
|
args.push("--keep-last", String(options.keepLast));
|
||||||
|
}
|
||||||
|
if (options.keepHourly) {
|
||||||
|
args.push("--keep-hourly", String(options.keepHourly));
|
||||||
|
}
|
||||||
|
if (options.keepDaily) {
|
||||||
|
args.push("--keep-daily", String(options.keepDaily));
|
||||||
|
}
|
||||||
|
if (options.keepWeekly) {
|
||||||
|
args.push("--keep-weekly", String(options.keepWeekly));
|
||||||
|
}
|
||||||
|
if (options.keepMonthly) {
|
||||||
|
args.push("--keep-monthly", String(options.keepMonthly));
|
||||||
|
}
|
||||||
|
if (options.keepYearly) {
|
||||||
|
args.push("--keep-yearly", String(options.keepYearly));
|
||||||
|
}
|
||||||
|
if (options.keepWithinDuration) {
|
||||||
|
args.push("--keep-within-duration", options.keepWithinDuration);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!extra.dryRun) {
|
||||||
|
args.push("--prune");
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic forget failed: ${res.stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, res.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
const lines = res.stdout.split("\n").filter((line) => line.trim());
|
||||||
|
const result = extra.dryRun ? safeJsonParse<ResticForgetResponse>(lines.at(-1) ?? "[]") : null;
|
||||||
|
|
||||||
|
return { success: true, data: result };
|
||||||
|
};
|
||||||
43
app/server/utils/restic/commands/init.ts
Normal file
43
app/server/utils/restic/commands/init.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { config as appConfig } from "~/server/core/config";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
import { keyAdd } from "./key-add";
|
||||||
|
|
||||||
|
export const init = async (config: RepositoryConfig, organizationId: string, options?: { timeoutMs?: number }) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
|
||||||
|
logger.info(`Initializing restic repository at ${repoUrl}...`);
|
||||||
|
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
const args = ["init", "--repo", repoUrl];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env, timeout: options?.timeoutMs ?? 60000 });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic init failed: ${res.stderr}`);
|
||||||
|
return { success: false, error: res.stderr };
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Restic repository initialized: ${repoUrl}`);
|
||||||
|
|
||||||
|
if (appConfig.resticHostname) {
|
||||||
|
const keyResult = await keyAdd(config, organizationId, {
|
||||||
|
host: appConfig.resticHostname,
|
||||||
|
timeoutMs: options?.timeoutMs,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!keyResult.success) {
|
||||||
|
logger.warn(`Repository initialized but failed to add key with hostname: ${keyResult.error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, error: null };
|
||||||
|
};
|
||||||
42
app/server/utils/restic/commands/key-add.ts
Normal file
42
app/server/utils/restic/commands/key-add.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const keyAdd = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
organizationId: string,
|
||||||
|
options: { host: string; timeoutMs?: number },
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
|
||||||
|
logger.info(`Adding restic key with host "${options.host}" for repository at ${repoUrl}...`);
|
||||||
|
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
const args = [
|
||||||
|
"key",
|
||||||
|
"add",
|
||||||
|
"--repo",
|
||||||
|
repoUrl,
|
||||||
|
"--host",
|
||||||
|
options.host,
|
||||||
|
"--new-password-file",
|
||||||
|
env.RESTIC_PASSWORD_FILE,
|
||||||
|
];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env, timeout: options.timeoutMs ?? 60000 });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic key add failed: ${res.stderr}`);
|
||||||
|
return { success: false, error: res.stderr };
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Restic key added with host "${options.host}" for repository: ${repoUrl}`);
|
||||||
|
return { success: true, error: null };
|
||||||
|
};
|
||||||
128
app/server/utils/restic/commands/ls.ts
Normal file
128
app/server/utils/restic/commands/ls.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
import { type } from "arktype";
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeSpawn } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
const lsNodeSchema = type({
|
||||||
|
name: "string",
|
||||||
|
type: "string",
|
||||||
|
path: "string",
|
||||||
|
uid: "number?",
|
||||||
|
gid: "number?",
|
||||||
|
size: "number?",
|
||||||
|
mode: "number?",
|
||||||
|
mtime: "string?",
|
||||||
|
atime: "string?",
|
||||||
|
ctime: "string?",
|
||||||
|
struct_type: "'node'",
|
||||||
|
});
|
||||||
|
|
||||||
|
const lsSnapshotInfoSchema = type({
|
||||||
|
time: "string",
|
||||||
|
parent: "string?",
|
||||||
|
tree: "string",
|
||||||
|
paths: "string[]",
|
||||||
|
hostname: "string",
|
||||||
|
username: "string?",
|
||||||
|
id: "string",
|
||||||
|
short_id: "string",
|
||||||
|
struct_type: "'snapshot'",
|
||||||
|
message_type: "'snapshot'",
|
||||||
|
});
|
||||||
|
|
||||||
|
export const ls = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
snapshotId: string,
|
||||||
|
organizationId: string,
|
||||||
|
path?: string,
|
||||||
|
options?: { offset?: number; limit?: number },
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "ls", snapshotId, "--long"];
|
||||||
|
|
||||||
|
if (path) {
|
||||||
|
args.push(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
let snapshot: typeof lsSnapshotInfoSchema.infer | null = null;
|
||||||
|
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
||||||
|
let totalNodes = 0;
|
||||||
|
let isFirstLine = true;
|
||||||
|
let hasMore = false;
|
||||||
|
|
||||||
|
const offset = Math.max(options?.offset ?? 0, 0);
|
||||||
|
const limit = Math.min(Math.max(options?.limit ?? 500, 1), 500);
|
||||||
|
|
||||||
|
const res = await safeSpawn({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
onStdout: (line) => {
|
||||||
|
const trimmedLine = line.trim();
|
||||||
|
if (!trimmedLine) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(trimmedLine);
|
||||||
|
|
||||||
|
if (isFirstLine) {
|
||||||
|
isFirstLine = false;
|
||||||
|
const snapshotValidation = lsSnapshotInfoSchema(data);
|
||||||
|
if (!(snapshotValidation instanceof type.errors)) {
|
||||||
|
snapshot = snapshotValidation;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodeValidation = lsNodeSchema(data);
|
||||||
|
if (nodeValidation instanceof type.errors) {
|
||||||
|
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalNodes >= offset && totalNodes < offset + limit) {
|
||||||
|
nodes.push(nodeValidation);
|
||||||
|
}
|
||||||
|
totalNodes++;
|
||||||
|
|
||||||
|
if (totalNodes >= offset + limit + 1) {
|
||||||
|
hasMore = true;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore JSON parse errors for non-JSON lines
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic ls failed: ${res.error}`);
|
||||||
|
throw new ResticError(res.exitCode, res.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalNodes > offset + limit) {
|
||||||
|
hasMore = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null,
|
||||||
|
nodes,
|
||||||
|
pagination: {
|
||||||
|
offset,
|
||||||
|
limit,
|
||||||
|
total: totalNodes,
|
||||||
|
hasMore,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
46
app/server/utils/restic/commands/repair-index.ts
Normal file
46
app/server/utils/restic/commands/repair-index.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const repairIndex = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
options: { signal?: AbortSignal; organizationId: string },
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
const args = ["repair", "index", "--repo", repoUrl];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (options.signal?.aborted) {
|
||||||
|
logger.warn("Restic repair index was aborted by signal.");
|
||||||
|
return { success: false, message: "Operation aborted", output: "" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { stdout, stderr } = res;
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic repair index failed: ${stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Restic repair index completed for repository: ${repoUrl}`);
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
output: stdout,
|
||||||
|
message: "Index repaired successfully",
|
||||||
|
};
|
||||||
|
};
|
||||||
161
app/server/utils/restic/commands/restore.ts
Normal file
161
app/server/utils/restic/commands/restore.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
import path from "node:path";
|
||||||
|
import { type } from "arktype";
|
||||||
|
import { throttle } from "es-toolkit";
|
||||||
|
import type { OverwriteMode, RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import type { ResticRestoreOutputDto } from "~/schemas/restic-dto";
|
||||||
|
import { resticRestoreOutputSchema } from "~/schemas/restic-dto";
|
||||||
|
import { findCommonAncestor } from "~/utils/common-ancestor";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeSpawn } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
const restoreProgressSchema = type({
|
||||||
|
message_type: "'status' | 'summary'",
|
||||||
|
seconds_elapsed: "number",
|
||||||
|
percent_done: "number = 0",
|
||||||
|
total_files: "number",
|
||||||
|
files_restored: "number = 0",
|
||||||
|
total_bytes: "number = 0",
|
||||||
|
bytes_restored: "number = 0",
|
||||||
|
});
|
||||||
|
|
||||||
|
export type RestoreProgress = typeof restoreProgressSchema.infer;
|
||||||
|
|
||||||
|
export const restore = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
snapshotId: string,
|
||||||
|
target: string,
|
||||||
|
options: {
|
||||||
|
basePath?: string;
|
||||||
|
organizationId: string;
|
||||||
|
include?: string[];
|
||||||
|
exclude?: string[];
|
||||||
|
excludeXattr?: string[];
|
||||||
|
delete?: boolean;
|
||||||
|
overwrite?: OverwriteMode;
|
||||||
|
onProgress?: (progress: RestoreProgress) => void;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
},
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
let restoreArg = snapshotId;
|
||||||
|
|
||||||
|
const includes = options.include?.length ? options.include : [options.basePath ?? "/"];
|
||||||
|
const commonAncestor = findCommonAncestor(includes);
|
||||||
|
if (target !== "/") {
|
||||||
|
restoreArg = `${snapshotId}:${commonAncestor}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const args = ["--repo", repoUrl, "restore", restoreArg, "--target", target];
|
||||||
|
|
||||||
|
if (options.overwrite) {
|
||||||
|
args.push("--overwrite", options.overwrite);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.include?.length) {
|
||||||
|
if (target === "/") {
|
||||||
|
for (const pattern of options.include) {
|
||||||
|
args.push("--include", pattern);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const strippedIncludes = options.include.map((pattern) => path.relative(commonAncestor, pattern));
|
||||||
|
const includesCoverRestoreRoot = strippedIncludes.some((pattern) => pattern === "" || pattern === ".");
|
||||||
|
|
||||||
|
if (!includesCoverRestoreRoot) {
|
||||||
|
for (const pattern of strippedIncludes) {
|
||||||
|
if (pattern !== "" && pattern !== ".") {
|
||||||
|
args.push("--include", pattern);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.exclude && options.exclude.length > 0) {
|
||||||
|
for (const pattern of options.exclude) {
|
||||||
|
args.push("--exclude", pattern);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.excludeXattr && options.excludeXattr.length > 0) {
|
||||||
|
for (const xattr of options.excludeXattr) {
|
||||||
|
args.push("--exclude-xattr", xattr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const streamProgress = throttle((data: string) => {
|
||||||
|
if (options.onProgress) {
|
||||||
|
try {
|
||||||
|
const jsonData = JSON.parse(data);
|
||||||
|
const progress = restoreProgressSchema(jsonData);
|
||||||
|
if (!(progress instanceof type.errors)) {
|
||||||
|
options.onProgress(progress);
|
||||||
|
} else {
|
||||||
|
logger.error(progress.summary);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore JSON parse errors for non-JSON lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 1000);
|
||||||
|
|
||||||
|
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||||
|
const res = await safeSpawn({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: options.signal,
|
||||||
|
onStdout: (data) => {
|
||||||
|
if (options.onProgress) {
|
||||||
|
streamProgress(data);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic restore failed: ${res.error}`);
|
||||||
|
throw new ResticError(res.exitCode, res.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastLine = res.summary.trim();
|
||||||
|
let summaryLine: unknown = {};
|
||||||
|
try {
|
||||||
|
summaryLine = JSON.parse(lastLine ?? "{}");
|
||||||
|
} catch {
|
||||||
|
logger.warn("Failed to parse restic restore output JSON summary.", lastLine);
|
||||||
|
summaryLine = {};
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`Restic restore output last line: ${JSON.stringify(summaryLine)}`);
|
||||||
|
const result = resticRestoreOutputSchema(summaryLine);
|
||||||
|
|
||||||
|
if (result instanceof type.errors) {
|
||||||
|
logger.warn(`Restic restore output validation failed: ${result.summary}`);
|
||||||
|
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`);
|
||||||
|
const fallback: ResticRestoreOutputDto = {
|
||||||
|
message_type: "summary" as const,
|
||||||
|
total_files: 0,
|
||||||
|
files_restored: 0,
|
||||||
|
files_skipped: 0,
|
||||||
|
bytes_skipped: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
`Restic restore completed for snapshot ${snapshotId} to target ${target}: ${result.files_restored} restored, ${result.files_skipped} skipped`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
58
app/server/utils/restic/commands/snapshots.ts
Normal file
58
app/server/utils/restic/commands/snapshots.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
import { type } from "arktype";
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { resticSnapshotSummarySchema } from "~/schemas/restic-dto";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
const snapshotInfoSchema = type({
|
||||||
|
gid: "number?",
|
||||||
|
hostname: "string",
|
||||||
|
id: "string",
|
||||||
|
parent: "string?",
|
||||||
|
paths: "string[]",
|
||||||
|
program_version: "string?",
|
||||||
|
short_id: "string",
|
||||||
|
time: "string",
|
||||||
|
uid: "number?",
|
||||||
|
username: "string?",
|
||||||
|
tags: "string[]?",
|
||||||
|
summary: resticSnapshotSummarySchema.optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; organizationId: string }) => {
|
||||||
|
const { tags, organizationId } = options;
|
||||||
|
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
const args = ["--repo", repoUrl, "snapshots"];
|
||||||
|
|
||||||
|
if (tags && tags.length > 0) {
|
||||||
|
for (const tag of tags) {
|
||||||
|
args.push("--tag", tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic snapshots retrieval failed: ${res.stderr}`);
|
||||||
|
throw new Error(`Restic snapshots retrieval failed: ${res.stderr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = snapshotInfoSchema.array()(JSON.parse(res.stdout));
|
||||||
|
|
||||||
|
if (result instanceof type.errors) {
|
||||||
|
logger.error(`Restic snapshots output validation failed: ${result.summary}`);
|
||||||
|
throw new Error(`Restic snapshots output validation failed: ${result.summary}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
37
app/server/utils/restic/commands/stats.ts
Normal file
37
app/server/utils/restic/commands/stats.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
import { type } from "arktype";
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { resticStatsSchema } from "~/schemas/restic-dto";
|
||||||
|
import { safeJsonParse } from "~/server/utils/json";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const stats = async (config: RepositoryConfig, options: { organizationId: string }) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
const args = ["--repo", repoUrl, "stats", "--mode", "raw-data"];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic stats retrieval failed: ${res.stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, res.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsedJson = safeJsonParse<unknown>(res.stdout);
|
||||||
|
const result = resticStatsSchema(parsedJson);
|
||||||
|
|
||||||
|
if (result instanceof type.errors) {
|
||||||
|
logger.error(`Restic stats output validation failed: ${result.summary}`);
|
||||||
|
throw new Error(`Restic stats output validation failed: ${result.summary}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
54
app/server/utils/restic/commands/tag-snapshots.ts
Normal file
54
app/server/utils/restic/commands/tag-snapshots.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const tagSnapshots = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
snapshotIds: string[],
|
||||||
|
tags: { add?: string[]; remove?: string[]; set?: string[] },
|
||||||
|
organizationId: string,
|
||||||
|
) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
if (snapshotIds.length === 0) {
|
||||||
|
throw new Error("No snapshot IDs provided for tagging.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const args: string[] = ["--repo", repoUrl, "tag", ...snapshotIds];
|
||||||
|
|
||||||
|
if (tags.add) {
|
||||||
|
for (const tag of tags.add) {
|
||||||
|
args.push("--add", tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tags.remove) {
|
||||||
|
for (const tag of tags.remove) {
|
||||||
|
args.push("--remove", tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tags.set) {
|
||||||
|
for (const tag of tags.set) {
|
||||||
|
args.push("--set", tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({ command: "restic", args, env });
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic snapshot tagging failed: ${res.stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, res.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true };
|
||||||
|
};
|
||||||
37
app/server/utils/restic/commands/unlock.ts
Normal file
37
app/server/utils/restic/commands/unlock.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { ResticError } from "~/server/utils/errors";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import { safeExec } from "~/server/utils/spawn";
|
||||||
|
import { addCommonArgs } from "../helpers/add-common-args";
|
||||||
|
import { buildEnv } from "../helpers/build-env";
|
||||||
|
import { buildRepoUrl } from "../helpers/build-repo-url";
|
||||||
|
import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
|
||||||
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
const env = await buildEnv(config, options.organizationId);
|
||||||
|
|
||||||
|
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
|
||||||
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
|
const res = await safeExec({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
|
if (options.signal?.aborted) {
|
||||||
|
logger.warn("Restic unlock was aborted by signal.");
|
||||||
|
return { success: false, message: "Operation aborted" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (res.exitCode !== 0) {
|
||||||
|
logger.error(`Restic unlock failed: ${res.stderr}`);
|
||||||
|
throw new ResticError(res.exitCode, res.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Restic unlock succeeded for repository: ${repoUrl}`);
|
||||||
|
return { success: true, message: "Repository unlocked successfully" };
|
||||||
|
};
|
||||||
314
app/server/utils/restic/helpers/__tests__/build-env.test.ts
Normal file
314
app/server/utils/restic/helpers/__tests__/build-env.test.ts
Normal file
|
|
@ -0,0 +1,314 @@
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import { db } from "~/server/db/db";
|
||||||
|
import { organization } from "~/server/db/schema";
|
||||||
|
import { RESTIC_CACHE_DIR } from "~/server/core/constants";
|
||||||
|
import { buildEnv } from "../build-env";
|
||||||
|
|
||||||
|
const withCustomPassword = <T extends object>(config: T) => ({
|
||||||
|
...config,
|
||||||
|
isExistingRepository: true as const,
|
||||||
|
customPassword: "test-password",
|
||||||
|
});
|
||||||
|
|
||||||
|
const createTestOrg = async (overrides: Partial<typeof organization.$inferInsert> = {}) => {
|
||||||
|
const id = randomUUID();
|
||||||
|
await db.insert(organization).values({
|
||||||
|
id,
|
||||||
|
name: "Build-env test org",
|
||||||
|
slug: `build-env-test-${id}`,
|
||||||
|
createdAt: new Date(),
|
||||||
|
metadata: { resticPassword: "org-restic-password" },
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
return id;
|
||||||
|
};
|
||||||
|
|
||||||
|
const PLAIN_PRIVATE_KEY =
|
||||||
|
"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA\n-----END OPENSSH PRIVATE KEY-----";
|
||||||
|
|
||||||
|
describe("buildEnv", () => {
|
||||||
|
describe("base environment", () => {
|
||||||
|
test("always sets RESTIC_CACHE_DIR", async () => {
|
||||||
|
const env = await buildEnv(withCustomPassword({ backend: "local" as const, path: "/tmp/repo" }), "org-1");
|
||||||
|
|
||||||
|
expect(env.RESTIC_CACHE_DIR).toBe(RESTIC_CACHE_DIR);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("always sets PATH", async () => {
|
||||||
|
const env = await buildEnv(withCustomPassword({ backend: "local" as const, path: "/tmp/repo" }), "org-1");
|
||||||
|
|
||||||
|
expect(env.PATH).toBeTruthy();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("password resolution", () => {
|
||||||
|
test("writes a password file when using customPassword on an existing repository", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
{ backend: "local" as const, path: "/tmp/repo", isExistingRepository: true, customPassword: "my-secret" },
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.RESTIC_PASSWORD_FILE).toMatch(/^\/tmp\/zerobyte-pass-.+\.txt$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("writes a password file from the organization's resticPassword when no customPassword is given", async () => {
|
||||||
|
const orgId = await createTestOrg();
|
||||||
|
|
||||||
|
const env = await buildEnv({ backend: "local" as const, path: "/tmp/repo" }, orgId);
|
||||||
|
|
||||||
|
expect(env.RESTIC_PASSWORD_FILE).toMatch(/^\/tmp\/zerobyte-pass-.+\.txt$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("throws when the organization does not exist", async () => {
|
||||||
|
await expect(buildEnv({ backend: "local" as const, path: "/tmp/repo" }, "non-existent-org")).rejects.toThrow(
|
||||||
|
"Organization non-existent-org not found",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("throws when the organization has no resticPassword configured", async () => {
|
||||||
|
const orgId = await createTestOrg({ metadata: null });
|
||||||
|
|
||||||
|
await expect(buildEnv({ backend: "local" as const, path: "/tmp/repo" }, orgId)).rejects.toThrow(
|
||||||
|
`Restic password not configured for organization ${orgId}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("s3 backend", () => {
|
||||||
|
const base = withCustomPassword({
|
||||||
|
backend: "s3" as const,
|
||||||
|
endpoint: "https://s3.amazonaws.com",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
accessKeyId: "my-access-key",
|
||||||
|
secretAccessKey: "my-secret-key",
|
||||||
|
});
|
||||||
|
|
||||||
|
test("sets AWS credentials", async () => {
|
||||||
|
const env = await buildEnv(base, "org-1");
|
||||||
|
|
||||||
|
expect(env.AWS_ACCESS_KEY_ID).toBe("my-access-key");
|
||||||
|
expect(env.AWS_SECRET_ACCESS_KEY).toBe("my-secret-key");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("sets AWS_S3_BUCKET_LOOKUP=dns for Huawei Cloud endpoints", async () => {
|
||||||
|
const env = await buildEnv({ ...base, endpoint: "https://obs.ap-southeast-1.myhuaweicloud.com" }, "org-1");
|
||||||
|
|
||||||
|
expect(env.AWS_S3_BUCKET_LOOKUP).toBe("dns");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not set AWS_S3_BUCKET_LOOKUP for standard S3 endpoints", async () => {
|
||||||
|
const env = await buildEnv(base, "org-1");
|
||||||
|
|
||||||
|
expect(env.AWS_S3_BUCKET_LOOKUP).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("r2 backend", () => {
|
||||||
|
test("sets AWS credentials with auto region and forced path style", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({
|
||||||
|
backend: "r2" as const,
|
||||||
|
endpoint: "https://myaccount.r2.cloudflarestorage.com",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
accessKeyId: "r2-access-key",
|
||||||
|
secretAccessKey: "r2-secret-key",
|
||||||
|
}),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.AWS_ACCESS_KEY_ID).toBe("r2-access-key");
|
||||||
|
expect(env.AWS_SECRET_ACCESS_KEY).toBe("r2-secret-key");
|
||||||
|
expect(env.AWS_REGION).toBe("auto");
|
||||||
|
expect(env.AWS_S3_FORCE_PATH_STYLE).toBe("true");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("gcs backend", () => {
|
||||||
|
test("sets project ID and writes credentials file", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({
|
||||||
|
backend: "gcs" as const,
|
||||||
|
bucket: "my-gcs-bucket",
|
||||||
|
projectId: "my-gcp-project",
|
||||||
|
credentialsJson: '{"type":"service_account"}',
|
||||||
|
}),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.GOOGLE_PROJECT_ID).toBe("my-gcp-project");
|
||||||
|
expect(env.GOOGLE_APPLICATION_CREDENTIALS).toMatch(/^\/tmp\/zerobyte-gcs-.+\.json$/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("azure backend", () => {
|
||||||
|
const base = withCustomPassword({
|
||||||
|
backend: "azure" as const,
|
||||||
|
container: "my-container",
|
||||||
|
accountName: "mystorageaccount",
|
||||||
|
accountKey: "my-account-key",
|
||||||
|
});
|
||||||
|
|
||||||
|
test("sets account name and key", async () => {
|
||||||
|
const env = await buildEnv(base, "org-1");
|
||||||
|
|
||||||
|
expect(env.AZURE_ACCOUNT_NAME).toBe("mystorageaccount");
|
||||||
|
expect(env.AZURE_ACCOUNT_KEY).toBe("my-account-key");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("includes endpoint suffix when provided", async () => {
|
||||||
|
const env = await buildEnv({ ...base, endpointSuffix: "core.chinacloudapi.cn" }, "org-1");
|
||||||
|
|
||||||
|
expect(env.AZURE_ENDPOINT_SUFFIX).toBe("core.chinacloudapi.cn");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits endpoint suffix when not provided", async () => {
|
||||||
|
const env = await buildEnv(base, "org-1");
|
||||||
|
|
||||||
|
expect(env.AZURE_ENDPOINT_SUFFIX).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("rest backend", () => {
|
||||||
|
test("sets username and password when both are provided", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({
|
||||||
|
backend: "rest" as const,
|
||||||
|
url: "https://rest-server.example.com",
|
||||||
|
username: "rest-user",
|
||||||
|
password: "rest-pass",
|
||||||
|
}),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.RESTIC_REST_USERNAME).toBe("rest-user");
|
||||||
|
expect(env.RESTIC_REST_PASSWORD).toBe("rest-pass");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits REST credentials when neither username nor password is provided", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({ backend: "rest" as const, url: "https://rest-server.example.com" }),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.RESTIC_REST_USERNAME).toBeUndefined();
|
||||||
|
expect(env.RESTIC_REST_PASSWORD).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sftp backend", () => {
|
||||||
|
const baseSftpConfig = withCustomPassword({
|
||||||
|
backend: "sftp" as const,
|
||||||
|
host: "backup.example.com",
|
||||||
|
port: 22,
|
||||||
|
user: "backup",
|
||||||
|
path: "/backups",
|
||||||
|
privateKey: PLAIN_PRIVATE_KEY,
|
||||||
|
skipHostKeyCheck: true as const,
|
||||||
|
});
|
||||||
|
|
||||||
|
test("throws for passphrase-protected private keys", async () => {
|
||||||
|
await expect(
|
||||||
|
buildEnv(
|
||||||
|
{
|
||||||
|
...baseSftpConfig,
|
||||||
|
privateKey: "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\n-----END RSA PRIVATE KEY-----",
|
||||||
|
},
|
||||||
|
"org-1",
|
||||||
|
),
|
||||||
|
).rejects.toThrow("Passphrase-protected SSH keys are not supported");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("succeeds when the private key has CRLF line endings", async () => {
|
||||||
|
const crlfKey = PLAIN_PRIVATE_KEY.replace(/\n/g, "\r\n");
|
||||||
|
|
||||||
|
await expect(buildEnv({ ...baseSftpConfig, privateKey: crlfKey }, "org-1")).resolves.toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses StrictHostKeyChecking=no when skipHostKeyCheck is true", async () => {
|
||||||
|
const env = await buildEnv({ ...baseSftpConfig, skipHostKeyCheck: true }, "org-1");
|
||||||
|
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("StrictHostKeyChecking=no");
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("UserKnownHostsFile=/dev/null");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses StrictHostKeyChecking=no when knownHosts is absent", async () => {
|
||||||
|
const env = await buildEnv({ ...baseSftpConfig, skipHostKeyCheck: false, knownHosts: undefined }, "org-1");
|
||||||
|
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("StrictHostKeyChecking=no");
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("UserKnownHostsFile=/dev/null");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses StrictHostKeyChecking=yes and a known hosts file when knownHosts is provided", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
{
|
||||||
|
...baseSftpConfig,
|
||||||
|
skipHostKeyCheck: false,
|
||||||
|
knownHosts: "backup.example.com ssh-rsa AAAAB3NzaC1...",
|
||||||
|
},
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("StrictHostKeyChecking=yes");
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("UserKnownHostsFile=/tmp/zerobyte-known-hosts-");
|
||||||
|
expect(env._SFTP_KNOWN_HOSTS_PATH).toMatch(/^\/tmp\/zerobyte-known-hosts-/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("adds -p flag for non-default ports", async () => {
|
||||||
|
const env = await buildEnv({ ...baseSftpConfig, port: 2222 }, "org-1");
|
||||||
|
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain("-p 2222");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("omits -p flag for the default port 22", async () => {
|
||||||
|
const env = await buildEnv({ ...baseSftpConfig, port: 22 }, "org-1");
|
||||||
|
|
||||||
|
expect(env._SFTP_SSH_ARGS).not.toContain("-p 22");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("sets the key path in both _SFTP_KEY_PATH and SSH args -i flag", async () => {
|
||||||
|
const env = await buildEnv(baseSftpConfig, "org-1");
|
||||||
|
|
||||||
|
expect(env._SFTP_KEY_PATH).toMatch(/^\/tmp\/zerobyte-ssh-/);
|
||||||
|
expect(env._SFTP_SSH_ARGS).toContain(`-i ${env._SFTP_KEY_PATH}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("cacert", () => {
|
||||||
|
test("sets RESTIC_CACERT pointing to a temp file when cacert is provided", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({
|
||||||
|
backend: "local" as const,
|
||||||
|
path: "/tmp/repo",
|
||||||
|
cacert: "-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----",
|
||||||
|
}),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env.RESTIC_CACERT).toMatch(/^\/tmp\/zerobyte-cacert-.+\.pem$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not set RESTIC_CACERT when cacert is absent", async () => {
|
||||||
|
const env = await buildEnv(withCustomPassword({ backend: "local" as const, path: "/tmp/repo" }), "org-1");
|
||||||
|
|
||||||
|
expect(env.RESTIC_CACERT).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("insecure TLS", () => {
|
||||||
|
test("sets _INSECURE_TLS=true when insecureTls is true", async () => {
|
||||||
|
const env = await buildEnv(
|
||||||
|
withCustomPassword({ backend: "local" as const, path: "/tmp/repo", insecureTls: true }),
|
||||||
|
"org-1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(env._INSECURE_TLS).toBe("true");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not set _INSECURE_TLS when insecureTls is absent", async () => {
|
||||||
|
const env = await buildEnv(withCustomPassword({ backend: "local" as const, path: "/tmp/repo" }), "org-1");
|
||||||
|
|
||||||
|
expect(env._INSECURE_TLS).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import { buildRepoUrl } from "../build-repo-url";
|
||||||
|
|
||||||
|
describe("buildRepoUrl", () => {
|
||||||
|
describe("S3 backend", () => {
|
||||||
|
test.each([
|
||||||
|
{
|
||||||
|
label: "standard endpoint",
|
||||||
|
endpoint: "https://s3.amazonaws.com",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
expected: "s3:https://s3.amazonaws.com/my-bucket",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "trailing slash on endpoint",
|
||||||
|
endpoint: "https://s3.xxxxxxxxx.net/",
|
||||||
|
bucket: "backup",
|
||||||
|
expected: "s3:https://s3.xxxxxxxxx.net/backup",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "trailing whitespace on endpoint",
|
||||||
|
endpoint: "https://s3.amazonaws.com/ ",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
expected: "s3:https://s3.amazonaws.com/my-bucket",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "leading and trailing whitespace on endpoint",
|
||||||
|
endpoint: " https://s3.amazonaws.com/ ",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
expected: "s3:https://s3.amazonaws.com/my-bucket",
|
||||||
|
},
|
||||||
|
])("$label → $expected", ({ endpoint, bucket, expected }) => {
|
||||||
|
expect(buildRepoUrl({ backend: "s3", endpoint, bucket, accessKeyId: "test", secretAccessKey: "test" })).toBe(
|
||||||
|
expected,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("R2 backend", () => {
|
||||||
|
test.each([
|
||||||
|
{
|
||||||
|
label: "standard endpoint (strips https:// protocol)",
|
||||||
|
endpoint: "https://myaccount.r2.cloudflarestorage.com",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
expected: "s3:myaccount.r2.cloudflarestorage.com/my-bucket",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "trailing slash on endpoint",
|
||||||
|
endpoint: "https://myaccount.r2.cloudflarestorage.com/",
|
||||||
|
bucket: "backup",
|
||||||
|
expected: "s3:myaccount.r2.cloudflarestorage.com/backup",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "leading and trailing whitespace on endpoint",
|
||||||
|
endpoint: " https://myaccount.r2.cloudflarestorage.com/ ",
|
||||||
|
bucket: "my-bucket",
|
||||||
|
expected: "s3:myaccount.r2.cloudflarestorage.com/my-bucket",
|
||||||
|
},
|
||||||
|
])("$label → $expected", ({ endpoint, bucket, expected }) => {
|
||||||
|
expect(buildRepoUrl({ backend: "r2", endpoint, bucket, accessKeyId: "test", secretAccessKey: "test" })).toBe(
|
||||||
|
expected,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("other backends", () => {
|
||||||
|
test("builds local repository URL", () => {
|
||||||
|
expect(buildRepoUrl({ backend: "local", path: "/path/to/repo" })).toBe("/path/to/repo");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
42
app/server/utils/restic/helpers/add-common-args.ts
Normal file
42
app/server/utils/restic/helpers/add-common-args.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { formatBandwidthLimit } from "./bandwidth";
|
||||||
|
import type { ResticEnv } from "../types";
|
||||||
|
|
||||||
|
export const addCommonArgs = (
|
||||||
|
args: string[],
|
||||||
|
env: ResticEnv,
|
||||||
|
config?: RepositoryConfig,
|
||||||
|
options?: { skipBandwidth?: boolean; includeJson?: boolean },
|
||||||
|
) => {
|
||||||
|
if (options?.includeJson !== false) {
|
||||||
|
args.push("--json");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env._SFTP_SSH_ARGS) {
|
||||||
|
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env.AWS_S3_BUCKET_LOOKUP === "dns") {
|
||||||
|
args.push("-o", "s3.bucket-lookup=dns");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env._INSECURE_TLS === "true") {
|
||||||
|
args.push("--insecure-tls");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env.RESTIC_CACERT) {
|
||||||
|
args.push("--cacert", env.RESTIC_CACERT);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config && !options?.skipBandwidth) {
|
||||||
|
const uploadLimit = formatBandwidthLimit(config.uploadLimit);
|
||||||
|
if (uploadLimit) {
|
||||||
|
args.push("--limit-upload", uploadLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
const downloadLimit = formatBandwidthLimit(config.downloadLimit);
|
||||||
|
if (downloadLimit) {
|
||||||
|
args.push("--limit-download", downloadLimit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
25
app/server/utils/restic/helpers/bandwidth.ts
Normal file
25
app/server/utils/restic/helpers/bandwidth.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
import type { BandwidthLimit } from "~/schemas/restic";
|
||||||
|
|
||||||
|
export const formatBandwidthLimit = (limit?: BandwidthLimit): string => {
|
||||||
|
if (!limit || !limit.enabled || limit.value <= 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
let kibibytesPerSecond: number;
|
||||||
|
switch (limit.unit) {
|
||||||
|
case "Kbps":
|
||||||
|
kibibytesPerSecond = (limit.value * 1000) / 8 / 1024;
|
||||||
|
break;
|
||||||
|
case "Mbps":
|
||||||
|
kibibytesPerSecond = (limit.value * 1000000) / (8 * 1024);
|
||||||
|
break;
|
||||||
|
case "Gbps":
|
||||||
|
kibibytesPerSecond = (limit.value * 1000000000) / (8 * 1024);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
const limitValue = Math.max(1, Math.floor(kibibytesPerSecond));
|
||||||
|
return `${limitValue}`;
|
||||||
|
};
|
||||||
167
app/server/utils/restic/helpers/build-env.ts
Normal file
167
app/server/utils/restic/helpers/build-env.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
import crypto from "node:crypto";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
import { RESTIC_CACHE_DIR } from "~/server/core/constants";
|
||||||
|
import { db } from "~/server/db/db";
|
||||||
|
import { cryptoUtils } from "~/server/utils/crypto";
|
||||||
|
import { logger } from "~/server/utils/logger";
|
||||||
|
import type { ResticEnv } from "../types";
|
||||||
|
|
||||||
|
export const buildEnv = async (config: RepositoryConfig, organizationId: string): Promise<ResticEnv> => {
|
||||||
|
const env: ResticEnv = {
|
||||||
|
RESTIC_CACHE_DIR,
|
||||||
|
PATH: process.env.PATH || "/usr/local/bin:/usr/bin:/bin",
|
||||||
|
};
|
||||||
|
|
||||||
|
if (config.isExistingRepository && config.customPassword) {
|
||||||
|
const decryptedPassword = await cryptoUtils.resolveSecret(config.customPassword);
|
||||||
|
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||||
|
|
||||||
|
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||||
|
mode: 0o600,
|
||||||
|
});
|
||||||
|
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||||
|
} else {
|
||||||
|
const org = await db.query.organization.findFirst({
|
||||||
|
where: { id: organizationId },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!org) {
|
||||||
|
throw new Error(`Organization ${organizationId} not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const metadata = org.metadata;
|
||||||
|
if (!metadata?.resticPassword) {
|
||||||
|
throw new Error(`Restic password not configured for organization ${organizationId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const decryptedPassword = await cryptoUtils.resolveSecret(metadata.resticPassword);
|
||||||
|
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||||
|
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||||
|
mode: 0o600,
|
||||||
|
});
|
||||||
|
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (config.backend) {
|
||||||
|
case "s3":
|
||||||
|
env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(config.accessKeyId);
|
||||||
|
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(config.secretAccessKey);
|
||||||
|
|
||||||
|
if (config.endpoint.includes("myhuaweicloud")) {
|
||||||
|
env.AWS_S3_BUCKET_LOOKUP = "dns";
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "r2":
|
||||||
|
env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(config.accessKeyId);
|
||||||
|
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(config.secretAccessKey);
|
||||||
|
env.AWS_REGION = "auto";
|
||||||
|
env.AWS_S3_FORCE_PATH_STYLE = "true";
|
||||||
|
break;
|
||||||
|
case "gcs": {
|
||||||
|
const decryptedCredentials = await cryptoUtils.resolveSecret(config.credentialsJson);
|
||||||
|
const credentialsPath = path.join("/tmp", `zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`);
|
||||||
|
await fs.writeFile(credentialsPath, decryptedCredentials, {
|
||||||
|
mode: 0o600,
|
||||||
|
});
|
||||||
|
env.GOOGLE_PROJECT_ID = config.projectId;
|
||||||
|
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "azure": {
|
||||||
|
env.AZURE_ACCOUNT_NAME = config.accountName;
|
||||||
|
env.AZURE_ACCOUNT_KEY = await cryptoUtils.resolveSecret(config.accountKey);
|
||||||
|
if (config.endpointSuffix) {
|
||||||
|
env.AZURE_ENDPOINT_SUFFIX = config.endpointSuffix;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "rest": {
|
||||||
|
if (config.username) {
|
||||||
|
env.RESTIC_REST_USERNAME = await cryptoUtils.resolveSecret(config.username);
|
||||||
|
}
|
||||||
|
if (config.password) {
|
||||||
|
env.RESTIC_REST_PASSWORD = await cryptoUtils.resolveSecret(config.password);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "sftp": {
|
||||||
|
const decryptedKey = await cryptoUtils.resolveSecret(config.privateKey);
|
||||||
|
const keyPath = path.join("/tmp", `zerobyte-ssh-${crypto.randomBytes(8).toString("hex")}`);
|
||||||
|
|
||||||
|
let normalizedKey = decryptedKey.replace(/\r\n/g, "\n");
|
||||||
|
if (!normalizedKey.endsWith("\n")) {
|
||||||
|
normalizedKey += "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedKey.includes("ENCRYPTED")) {
|
||||||
|
logger.error("SFTP: Private key appears to be passphrase-protected. Please use an unencrypted key.");
|
||||||
|
throw new Error("Passphrase-protected SSH keys are not supported. Please provide an unencrypted private key.");
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.writeFile(keyPath, normalizedKey, { mode: 0o600 });
|
||||||
|
|
||||||
|
env._SFTP_KEY_PATH = keyPath;
|
||||||
|
|
||||||
|
const sshArgs = [
|
||||||
|
"-o",
|
||||||
|
"LogLevel=ERROR",
|
||||||
|
"-o",
|
||||||
|
"BatchMode=yes",
|
||||||
|
"-o",
|
||||||
|
"NumberOfPasswordPrompts=0",
|
||||||
|
"-o",
|
||||||
|
"PreferredAuthentications=publickey",
|
||||||
|
"-o",
|
||||||
|
"PasswordAuthentication=no",
|
||||||
|
"-o",
|
||||||
|
"KbdInteractiveAuthentication=no",
|
||||||
|
"-o",
|
||||||
|
"IdentitiesOnly=yes",
|
||||||
|
"-o",
|
||||||
|
"ConnectTimeout=10",
|
||||||
|
"-o",
|
||||||
|
"ConnectionAttempts=1",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveInterval=60",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveCountMax=240",
|
||||||
|
"-i",
|
||||||
|
keyPath,
|
||||||
|
];
|
||||||
|
|
||||||
|
if (config.skipHostKeyCheck || !config.knownHosts) {
|
||||||
|
sshArgs.push("-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null");
|
||||||
|
} else if (config.knownHosts) {
|
||||||
|
const knownHostsPath = path.join("/tmp", `zerobyte-known-hosts-${crypto.randomBytes(8).toString("hex")}`);
|
||||||
|
await fs.writeFile(knownHostsPath, config.knownHosts, {
|
||||||
|
mode: 0o600,
|
||||||
|
});
|
||||||
|
env._SFTP_KNOWN_HOSTS_PATH = knownHostsPath;
|
||||||
|
sshArgs.push("-o", "StrictHostKeyChecking=yes", "-o", `UserKnownHostsFile=${knownHostsPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.port && config.port !== 22) {
|
||||||
|
sshArgs.push("-p", String(config.port));
|
||||||
|
}
|
||||||
|
|
||||||
|
env._SFTP_SSH_ARGS = sshArgs.join(" ");
|
||||||
|
logger.info(`SFTP: SSH args: ${env._SFTP_SSH_ARGS}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.cacert) {
|
||||||
|
const decryptedCert = await cryptoUtils.resolveSecret(config.cacert);
|
||||||
|
const certPath = path.join("/tmp", `zerobyte-cacert-${crypto.randomBytes(8).toString("hex")}.pem`);
|
||||||
|
await fs.writeFile(certPath, decryptedCert, { mode: 0o600 });
|
||||||
|
env.RESTIC_CACERT = certPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.insecureTls) {
|
||||||
|
env._INSECURE_TLS = "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
return env;
|
||||||
|
};
|
||||||
34
app/server/utils/restic/helpers/build-repo-url.ts
Normal file
34
app/server/utils/restic/helpers/build-repo-url.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
import type { RepositoryConfig } from "~/schemas/restic";
|
||||||
|
|
||||||
|
export const buildRepoUrl = (config: RepositoryConfig): string => {
|
||||||
|
switch (config.backend) {
|
||||||
|
case "local":
|
||||||
|
return config.path;
|
||||||
|
case "s3": {
|
||||||
|
const endpoint = config.endpoint.trim().replace(/\/$/, "");
|
||||||
|
return `s3:${endpoint}/${config.bucket}`;
|
||||||
|
}
|
||||||
|
case "r2": {
|
||||||
|
const endpoint = config.endpoint
|
||||||
|
.trim()
|
||||||
|
.replace(/^https?:\/\//, "")
|
||||||
|
.replace(/\/$/, "");
|
||||||
|
return `s3:${endpoint}/${config.bucket}`;
|
||||||
|
}
|
||||||
|
case "gcs":
|
||||||
|
return `gs:${config.bucket}:/`;
|
||||||
|
case "azure":
|
||||||
|
return `azure:${config.container}:/`;
|
||||||
|
case "rclone":
|
||||||
|
return `rclone:${config.remote}:${config.path}`;
|
||||||
|
case "rest": {
|
||||||
|
const pathSuffix = config.path ? `/${config.path}` : "";
|
||||||
|
return `rest:${config.url}${pathSuffix}`;
|
||||||
|
}
|
||||||
|
case "sftp":
|
||||||
|
return `sftp:${config.user}@${config.host}:${config.path}`;
|
||||||
|
default: {
|
||||||
|
throw new Error(`Unsupported repository backend: ${JSON.stringify(config)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
17
app/server/utils/restic/helpers/cleanup-temporary-keys.ts
Normal file
17
app/server/utils/restic/helpers/cleanup-temporary-keys.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import { RESTIC_PASS_FILE } from "~/server/core/constants";
|
||||||
|
import type { ResticEnv } from "../types";
|
||||||
|
|
||||||
|
export const cleanupTemporaryKeys = async (env: ResticEnv) => {
|
||||||
|
const keysToClean = ["_SFTP_KEY_PATH", "_SFTP_KNOWN_HOSTS_PATH", "RESTIC_CACERT", "GOOGLE_APPLICATION_CREDENTIALS"];
|
||||||
|
|
||||||
|
for (const key of keysToClean) {
|
||||||
|
if (env[key]) {
|
||||||
|
await fs.unlink(env[key]).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env.RESTIC_PASSWORD_FILE && env.RESTIC_PASSWORD_FILE !== RESTIC_PASS_FILE) {
|
||||||
|
await fs.unlink(env.RESTIC_PASSWORD_FILE).catch(() => {});
|
||||||
|
}
|
||||||
|
};
|
||||||
42
app/server/utils/restic/index.ts
Normal file
42
app/server/utils/restic/index.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
import { backup } from "./commands/backup";
|
||||||
|
import { check } from "./commands/check";
|
||||||
|
import { copy } from "./commands/copy";
|
||||||
|
import { deleteSnapshot, deleteSnapshots } from "./commands/delete-snapshots";
|
||||||
|
import { dump } from "./commands/dump";
|
||||||
|
import { forget } from "./commands/forget";
|
||||||
|
import { init } from "./commands/init";
|
||||||
|
import { keyAdd } from "./commands/key-add";
|
||||||
|
import { ls } from "./commands/ls";
|
||||||
|
import { repairIndex } from "./commands/repair-index";
|
||||||
|
import { restore } from "./commands/restore";
|
||||||
|
import { snapshots } from "./commands/snapshots";
|
||||||
|
import { stats } from "./commands/stats";
|
||||||
|
import { tagSnapshots } from "./commands/tag-snapshots";
|
||||||
|
import { unlock } from "./commands/unlock";
|
||||||
|
|
||||||
|
export { addCommonArgs } from "./helpers/add-common-args";
|
||||||
|
export { buildEnv } from "./helpers/build-env";
|
||||||
|
export { buildRepoUrl } from "./helpers/build-repo-url";
|
||||||
|
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
||||||
|
|
||||||
|
export type { RestoreProgress } from "./commands/restore";
|
||||||
|
export type { ForgetGroup, ForgetReason, ResticDumpStream, ResticForgetResponse, Snapshot } from "./types";
|
||||||
|
|
||||||
|
export const restic = {
|
||||||
|
init,
|
||||||
|
keyAdd,
|
||||||
|
backup,
|
||||||
|
restore,
|
||||||
|
dump,
|
||||||
|
snapshots,
|
||||||
|
stats,
|
||||||
|
forget,
|
||||||
|
deleteSnapshot,
|
||||||
|
deleteSnapshots,
|
||||||
|
tagSnapshots,
|
||||||
|
unlock,
|
||||||
|
ls,
|
||||||
|
check,
|
||||||
|
repairIndex,
|
||||||
|
copy,
|
||||||
|
};
|
||||||
43
app/server/utils/restic/types.ts
Normal file
43
app/server/utils/restic/types.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
import type { Readable } from "node:stream";
|
||||||
|
import type { ResticSnapshotSummaryDto } from "~/schemas/restic-dto";
|
||||||
|
|
||||||
|
export type ResticEnv = Record<string, string>;
|
||||||
|
|
||||||
|
export interface ResticDumpStream {
|
||||||
|
stream: Readable;
|
||||||
|
completion: Promise<void>;
|
||||||
|
abort: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ResticForgetResponse = ForgetGroup[];
|
||||||
|
|
||||||
|
export interface ForgetGroup {
|
||||||
|
tags: string[] | null;
|
||||||
|
host: string;
|
||||||
|
paths: string[] | null;
|
||||||
|
keep: Snapshot[];
|
||||||
|
remove: Snapshot[] | null;
|
||||||
|
reasons: ForgetReason[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Snapshot {
|
||||||
|
time: string;
|
||||||
|
parent?: string;
|
||||||
|
tree: string;
|
||||||
|
paths: string[];
|
||||||
|
hostname: string;
|
||||||
|
username?: string;
|
||||||
|
uid?: number;
|
||||||
|
gid?: number;
|
||||||
|
excludes?: string[] | null;
|
||||||
|
tags?: string[] | null;
|
||||||
|
program_version?: string;
|
||||||
|
summary?: ResticSnapshotSummaryDto;
|
||||||
|
id: string;
|
||||||
|
short_id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ForgetReason {
|
||||||
|
snapshot: Snapshot;
|
||||||
|
matches: string[];
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue