diff --git a/README.md b/README.md index af1a43cb..037e83aa 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Zerobyte can be customized using environment variables. Below are the available | `TRUSTED_ORIGINS` | Comma-separated list of extra trusted origins for CORS (e.g., `http://localhost:3000,http://example.com`). | (none) | | `LOG_LEVEL` | Logging verbosity. Options: `debug`, `info`, `warn`, `error`. | `info` | | `SERVER_IDLE_TIMEOUT` | Idle timeout for the server in seconds. | `60` | -| `RCLONE_CONFIG_DIR` | Path to the rclone config directory inside the container. Change this if running as a non-root user. | `/root/.config/rclone` | +| `RCLONE_CONFIG_DIR` | Path to the directory containing `rclone.conf` inside the container. Change this if running as a non-root user. | `/root/.config/rclone` | | `PROVISIONING_PATH` | Path to a JSON file with operator-managed repositories and volumes to sync at startup. | (none) | ### Provisioned Resources @@ -258,7 +258,7 @@ Zerobyte can use [rclone](https://rclone.org/) to support 40+ cloud storage prov + - ~/.config/rclone:/root/.config/rclone:ro ``` - > **Note for non-root users:** If your container runs as a different user (e.g., TrueNAS apps), mount your config to the appropriate location and set `RCLONE_CONF_DIR`: + > **Note for non-root users:** If your container runs as a different user (e.g., TrueNAS apps), mount your config to the appropriate location and set `RCLONE_CONFIG_DIR`: > > ```yaml > environment: diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index d8d305f4..e981e87c 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -283,9 +283,8 @@ If you're experiencing rclone issues, verify all of the following: **Diagnosis:** ```bash -# Check if config is accessible inside container -docker exec zerobyte ls -la /root/.config/rclone/ -docker exec zerobyte cat /root/.config/rclone/rclone.conf +# Check which config file rclone will use inside the container +docker exec zerobyte sh -lc 'echo HOME=$HOME; rclone config file' ``` **Solutions:** diff --git a/app/server/core/capabilities.ts b/app/server/core/capabilities.ts index ec1afc94..b5621acc 100644 --- a/app/server/core/capabilities.ts +++ b/app/server/core/capabilities.ts @@ -1,5 +1,5 @@ import * as fs from "node:fs/promises"; -import { RCLONE_CONFIG_DIR } from "./constants"; +import { RCLONE_CONFIG_DIR, RCLONE_CONFIG_FILE } from "./constants"; import { logger } from "@zerobyte/core/node"; export type SystemCapabilities = { @@ -35,17 +35,11 @@ async function detectCapabilities(): Promise { /** * Checks if rclone is available by: - * 1. Checking if the rclone config directory exists and is accessible + * 1. Checking if the rclone config file exists and is accessible */ async function detectRclone(): Promise { try { - await fs.access(RCLONE_CONFIG_DIR); - - // Make sure the folder is not empty - const files = await fs.readdir(RCLONE_CONFIG_DIR); - if (files.length === 0) { - throw new Error("rclone config directory is empty"); - } + await fs.access(RCLONE_CONFIG_FILE); logger.info("rclone capability: enabled"); return true; diff --git a/app/server/core/constants.ts b/app/server/core/constants.ts index acb10ebc..2adc4108 100644 --- a/app/server/core/constants.ts +++ b/app/server/core/constants.ts @@ -1,3 +1,5 @@ +import path from "node:path"; + export const OPERATION_TIMEOUT = 5000; export const VOLUME_MOUNT_BASE = process.env.ZEROBYTE_VOLUMES_DIR || "/var/lib/zerobyte/volumes"; @@ -10,6 +12,7 @@ export const RESTIC_PASS_FILE = process.env.RESTIC_PASS_FILE || "/var/lib/zeroby export const SSH_KEYS_DIR = "/var/lib/zerobyte/ssh"; export const RCLONE_CONFIG_DIR = process.env.RCLONE_CONFIG_DIR || "/root/.config/rclone"; +export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf"); export const RESTORE_BLOCKED_ROOTS = [REPOSITORY_BASE, RESTIC_CACHE_DIR, SSH_KEYS_DIR, RCLONE_CONFIG_DIR, "/app"]; export const DEFAULT_EXCLUDES = [RESTIC_PASS_FILE, REPOSITORY_BASE]; diff --git a/app/server/core/restic.ts b/app/server/core/restic.ts index 79f621d6..dfc857b4 100644 --- a/app/server/core/restic.ts +++ b/app/server/core/restic.ts @@ -1,6 +1,6 @@ import { createRestic } from "@zerobyte/core/restic/server"; import type { ResticDeps } from "@zerobyte/core/restic"; -import { DEFAULT_EXCLUDES, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "./constants"; +import { DEFAULT_EXCLUDES, RCLONE_CONFIG_FILE, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "./constants"; import { config } from "./config"; import { cryptoUtils } from "../utils/crypto"; import { db } from "../db/db"; @@ -26,6 +26,7 @@ export const resticDeps: ResticDeps = { resticCacheDir: RESTIC_CACHE_DIR, resticPassFile: RESTIC_PASS_FILE, defaultExcludes: DEFAULT_EXCLUDES, + rcloneConfigFile: RCLONE_CONFIG_FILE, hostname: config.resticHostname, }; diff --git a/app/server/modules/agents/__tests__/session.test.ts b/app/server/modules/agents/__tests__/session.test.ts index 87671f4e..b787e1ea 100644 --- a/app/server/modules/agents/__tests__/session.test.ts +++ b/app/server/modules/agents/__tests__/session.test.ts @@ -131,6 +131,7 @@ test("close emits a synthetic backup.cancelled for a queued backup", () => { cacheDir: "/tmp/cache", passFile: "/tmp/pass", defaultExcludes: [], + rcloneConfigFile: "/tmp/rclone.conf", }, }), ); diff --git a/app/server/modules/backends/rclone/rclone-backend.ts b/app/server/modules/backends/rclone/rclone-backend.ts index 06b946e0..d4d8d241 100644 --- a/app/server/modules/backends/rclone/rclone-backend.ts +++ b/app/server/modules/backends/rclone/rclone-backend.ts @@ -1,6 +1,6 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; -import { OPERATION_TIMEOUT } from "../../../core/constants"; +import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE } from "../../../core/constants"; import { toMessage } from "../../../utils/errors"; import { logger } from "@zerobyte/core/node"; import { getMountForPath } from "../../../utils/mountinfo"; @@ -51,7 +51,12 @@ const mount = async (config: BackendConfig, path: string) => { logger.debug(`Mounting rclone volume ${path}...`); logger.info(`Executing rclone: rclone ${args.join(" ")}`); - const result = await safeExec({ command: "rclone", args, timeout: zbConfig.serverIdleTimeout * 1000 }); + const result = await safeExec({ + command: "rclone", + args, + env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE }, + timeout: zbConfig.serverIdleTimeout * 1000, + }); if (result.exitCode !== 0) { const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error"; diff --git a/app/server/utils/__tests__/rclone.test.ts b/app/server/utils/__tests__/rclone.test.ts new file mode 100644 index 00000000..3787ea07 --- /dev/null +++ b/app/server/utils/__tests__/rclone.test.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; + +vi.mock("@zerobyte/core/node", async (importOriginal) => { + const actual = await importOriginal(); + + return { + ...actual, + safeExec: vi.fn(), + }; +}); + +import { safeExec } from "@zerobyte/core/node"; +import { RCLONE_CONFIG_FILE } from "../../core/constants"; +import { getRcloneRemoteInfo, listRcloneRemotes } from "../rclone"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("rclone utilities", () => { + test("lists remotes using the configured rclone config file", async () => { + vi.mocked(safeExec).mockResolvedValueOnce({ + exitCode: 0, + stdout: "primary:\narchive:\n", + stderr: "", + timedOut: false, + }); + + expect(await listRcloneRemotes()).toEqual(["primary", "archive"]); + expect(safeExec).toHaveBeenCalledWith({ + command: "rclone", + args: ["listremotes"], + env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE }, + }); + }); + + test("loads remote details using the configured rclone config file", async () => { + vi.mocked(safeExec).mockResolvedValueOnce({ + exitCode: 0, + stdout: "[primary]\ntype = s3\nprovider = AWS\n", + stderr: "", + timedOut: false, + }); + + expect(await getRcloneRemoteInfo("primary")).toEqual({ + type: "s3", + config: { + type: "s3", + provider: "AWS", + }, + }); + expect(safeExec).toHaveBeenCalledWith({ + command: "rclone", + args: ["config", "show", "primary"], + env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE }, + }); + }); +}); diff --git a/app/server/utils/rclone.ts b/app/server/utils/rclone.ts index 083b0740..1f00639a 100644 --- a/app/server/utils/rclone.ts +++ b/app/server/utils/rclone.ts @@ -1,13 +1,18 @@ import { logger } from "@zerobyte/core/node"; import { toMessage } from "./errors"; import { safeExec } from "@zerobyte/core/node"; +import { RCLONE_CONFIG_FILE } from "../core/constants"; + +const rcloneEnv = { + RCLONE_CONFIG: RCLONE_CONFIG_FILE, +}; /** * List all configured rclone remotes * @returns Array of remote names */ export async function listRcloneRemotes(): Promise { - const result = await safeExec({ command: "rclone", args: ["listremotes"] }); + const result = await safeExec({ command: "rclone", args: ["listremotes"], env: rcloneEnv }); if (result.exitCode !== 0) { logger.error(`Failed to list rclone remotes: ${result.stderr.toString()}`); @@ -34,7 +39,7 @@ export async function getRcloneRemoteInfo( remote: string, ): Promise<{ type: string; config: Record } | null> { try { - const result = await safeExec({ command: "rclone", args: ["config", "show", remote] }); + const result = await safeExec({ command: "rclone", args: ["config", "show", remote], env: rcloneEnv }); if (result.exitCode !== 0) { logger.error(`Failed to get info for remote ${remote}: ${result.stderr.toString()}`); diff --git a/apps/agent/src/__tests__/controller-session.test.ts b/apps/agent/src/__tests__/controller-session.test.ts index b4afda3b..e9ff4e83 100644 --- a/apps/agent/src/__tests__/controller-session.test.ts +++ b/apps/agent/src/__tests__/controller-session.test.ts @@ -44,6 +44,7 @@ test("emits backup.failed when a backup command hits a restic error", async () = cacheDir: "/tmp/restic-cache", passFile: "/tmp/restic-pass", defaultExcludes: [], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }, }), ); diff --git a/apps/agent/src/commands/backup-run.ts b/apps/agent/src/commands/backup-run.ts index a99b8f9f..56511d4e 100644 --- a/apps/agent/src/commands/backup-run.ts +++ b/apps/agent/src/commands/backup-run.ts @@ -50,6 +50,7 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa resticPassFile: payload.runtime.passFile, defaultExcludes: payload.runtime.defaultExcludes, hostname: payload.runtime.hostname, + rcloneConfigFile: payload.runtime.rcloneConfigFile, }; const restic = createRestic(deps); diff --git a/packages/contracts/src/agent-protocol.ts b/packages/contracts/src/agent-protocol.ts index ab6a99b2..3144514b 100644 --- a/packages/contracts/src/agent-protocol.ts +++ b/packages/contracts/src/agent-protocol.ts @@ -26,6 +26,7 @@ const backupRuntimeSchema = z.object({ passFile: z.string(), defaultExcludes: z.array(z.string()), hostname: z.string().optional(), + rcloneConfigFile: z.string(), }); const backupRunSchema = z.object({ diff --git a/packages/core/src/restic/commands/__tests__/backup.test.ts b/packages/core/src/restic/commands/__tests__/backup.test.ts index d1345f0f..8c23ac89 100644 --- a/packages/core/src/restic/commands/__tests__/backup.test.ts +++ b/packages/core/src/restic/commands/__tests__/backup.test.ts @@ -14,6 +14,7 @@ const mockDeps: ResticDeps = { resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], hostname: "zerobyte", + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const VALID_SUMMARY = JSON.stringify({ @@ -61,10 +62,12 @@ type SetupOptions = { */ const setup = ({ spawnResult = {}, onSpawnCall }: SetupOptions = {}) => { let capturedArgs: string[] = []; + let capturedEnv: SafeSpawnParams["env"]; vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve()); vi.spyOn(spawnModule, "safeSpawn").mockImplementation((params: SafeSpawnParams) => { capturedArgs = params.args; + capturedEnv = params.env; return Promise.resolve(onSpawnCall?.(params)).then(() => ({ exitCode: 0, summary: VALID_SUMMARY, @@ -75,6 +78,7 @@ const setup = ({ spawnResult = {}, onSpawnCall }: SetupOptions = {}) => { return { getArgs: () => capturedArgs, + getEnv: () => capturedEnv, hasFlag: (flag: string) => capturedArgs.includes(flag), getOptionValues: (option: string): string[] => { const values: string[] = []; @@ -160,6 +164,25 @@ describe("backup command", () => { expect(getOptionValues("--exclude").length).toBeGreaterThan(0); }); + + test("passes RCLONE_CONFIG when backing up to an rclone repository", async () => { + const { getEnv } = setup(); + + await runBackup( + { + backend: "rclone", + remote: "remote", + path: "/repo", + isExistingRepository: true, + customPassword: "custom-password", + }, + "/mnt/data", + { organizationId: "org-1" }, + mockDeps, + ); + + expect(getEnv()?.RCLONE_CONFIG).toBe(mockDeps.rcloneConfigFile); + }); }); describe("exit code handling", () => { diff --git a/packages/core/src/restic/commands/__tests__/copy.test.ts b/packages/core/src/restic/commands/__tests__/copy.test.ts index 3450902e..b7fbde5a 100644 --- a/packages/core/src/restic/commands/__tests__/copy.test.ts +++ b/packages/core/src/restic/commands/__tests__/copy.test.ts @@ -10,6 +10,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const sourceConfig = { diff --git a/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts b/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts index 052545b8..245f8424 100644 --- a/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts +++ b/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts @@ -10,6 +10,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const config = { diff --git a/packages/core/src/restic/commands/__tests__/dump.test.ts b/packages/core/src/restic/commands/__tests__/dump.test.ts index 9048fc02..ab329071 100644 --- a/packages/core/src/restic/commands/__tests__/dump.test.ts +++ b/packages/core/src/restic/commands/__tests__/dump.test.ts @@ -11,6 +11,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const config = { diff --git a/packages/core/src/restic/commands/__tests__/ls.test.ts b/packages/core/src/restic/commands/__tests__/ls.test.ts index ebe21c85..74d5a0aa 100644 --- a/packages/core/src/restic/commands/__tests__/ls.test.ts +++ b/packages/core/src/restic/commands/__tests__/ls.test.ts @@ -11,6 +11,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const config = { diff --git a/packages/core/src/restic/commands/__tests__/restore.test.ts b/packages/core/src/restic/commands/__tests__/restore.test.ts index ba731f44..1345ccb8 100644 --- a/packages/core/src/restic/commands/__tests__/restore.test.ts +++ b/packages/core/src/restic/commands/__tests__/restore.test.ts @@ -12,6 +12,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const successfulRestoreSummary = JSON.stringify({ diff --git a/packages/core/src/restic/commands/__tests__/snapshots.test.ts b/packages/core/src/restic/commands/__tests__/snapshots.test.ts index a2728208..6619a98e 100644 --- a/packages/core/src/restic/commands/__tests__/snapshots.test.ts +++ b/packages/core/src/restic/commands/__tests__/snapshots.test.ts @@ -11,6 +11,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const config = { diff --git a/packages/core/src/restic/commands/__tests__/tag-snapshots.test.ts b/packages/core/src/restic/commands/__tests__/tag-snapshots.test.ts index 8d412505..e028ec61 100644 --- a/packages/core/src/restic/commands/__tests__/tag-snapshots.test.ts +++ b/packages/core/src/restic/commands/__tests__/tag-snapshots.test.ts @@ -10,6 +10,7 @@ const mockDeps: ResticDeps = { resticCacheDir: "/tmp/restic-cache", resticPassFile: "/tmp/restic.pass", defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", }; const config = { diff --git a/packages/core/src/restic/helpers/__tests__/build-env.test.ts b/packages/core/src/restic/helpers/__tests__/build-env.test.ts index 6428af73..9d9ac21f 100644 --- a/packages/core/src/restic/helpers/__tests__/build-env.test.ts +++ b/packages/core/src/restic/helpers/__tests__/build-env.test.ts @@ -12,6 +12,7 @@ const makeDeps = (overrides: Partial = {}): ResticDeps => ({ resticCacheDir: RESTIC_CACHE_DIR, resticPassFile: RESTIC_PASS_FILE, defaultExcludes: [RESTIC_PASS_FILE, "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/tmp/rclone.conf", ...overrides, }); diff --git a/packages/core/src/restic/helpers/build-env.ts b/packages/core/src/restic/helpers/build-env.ts index ae6ed50e..1c479831 100644 --- a/packages/core/src/restic/helpers/build-env.ts +++ b/packages/core/src/restic/helpers/build-env.ts @@ -69,6 +69,9 @@ export const buildEnv = async ( } break; } + case "rclone": + env.RCLONE_CONFIG = deps.rcloneConfigFile; + break; case "sftp": { const decryptedKey = await deps.resolveSecret(config.privateKey); const keyPath = path.join("/tmp", `zerobyte-ssh-${crypto.randomBytes(8).toString("hex")}`); diff --git a/packages/core/src/restic/types.ts b/packages/core/src/restic/types.ts index cf68d862..68f503cb 100644 --- a/packages/core/src/restic/types.ts +++ b/packages/core/src/restic/types.ts @@ -7,6 +7,7 @@ export interface ResticDeps { resticCacheDir: string; resticPassFile: string; defaultExcludes: string[]; + rcloneConfigFile: string; hostname?: string; }