fix(rclone): pass explicitly the RCLONE_CONF env var (#779)

README.md

@@ -105,7 +105,7 @@ Zerobyte can be customized using environment variables. Below are the available
 | `TRUSTED_ORIGINS`     | Comma-separated list of extra trusted origins for CORS (e.g., `http://localhost:3000,http://example.com`).                                | (none)                 |
 | `LOG_LEVEL`           | Logging verbosity. Options: `debug`, `info`, `warn`, `error`.                                                                             | `info`                 |
 | `SERVER_IDLE_TIMEOUT` | Idle timeout for the server in seconds.                                                                                                   | `60`                   |
-| `RCLONE_CONFIG_DIR`   | Path to the rclone config directory inside the container. Change this if running as a non-root user.                                      | `/root/.config/rclone` |
+| `RCLONE_CONFIG_DIR`   | Path to the directory containing `rclone.conf` inside the container. Change this if running as a non-root user.                           | `/root/.config/rclone` |
 | `PROVISIONING_PATH`   | Path to a JSON file with operator-managed repositories and volumes to sync at startup.                                                    | (none)                 |
 
 ### Provisioned Resources
@@ -258,7 +258,7 @@ Zerobyte can use [rclone](https://rclone.org/) to support 40+ cloud storage prov
    +     - ~/.config/rclone:/root/.config/rclone:ro
    ```
 
-   > **Note for non-root users:** If your container runs as a different user (e.g., TrueNAS apps), mount your config to the appropriate location and set `RCLONE_CONF_DIR`:
+   > **Note for non-root users:** If your container runs as a different user (e.g., TrueNAS apps), mount your config to the appropriate location and set `RCLONE_CONFIG_DIR`:
    >
    > ```yaml
    > environment:

TROUBLESHOOTING.md

@@ -283,9 +283,8 @@ If you're experiencing rclone issues, verify all of the following:
 **Diagnosis:**
 
 ```bash
-# Check if config is accessible inside container
+# Check which config file rclone will use inside the container
-docker exec zerobyte ls -la /root/.config/rclone/
+docker exec zerobyte sh -lc 'echo HOME=$HOME; rclone config file'
-docker exec zerobyte cat /root/.config/rclone/rclone.conf
 ```
 
 **Solutions:**

app/server/core/capabilities.ts

@@ -1,5 +1,5 @@
 import * as fs from "node:fs/promises";
-import { RCLONE_CONFIG_DIR } from "./constants";
+import { RCLONE_CONFIG_DIR, RCLONE_CONFIG_FILE } from "./constants";
 import { logger } from "@zerobyte/core/node";
 
 export type SystemCapabilities = {
@@ -35,17 +35,11 @@ async function detectCapabilities(): Promise<SystemCapabilities> {
 
 /**
  * Checks if rclone is available by:
- * 1. Checking if the rclone config directory exists and is accessible
+ * 1. Checking if the rclone config file exists and is accessible
  */
 async function detectRclone(): Promise<boolean> {
 	try {
-		await fs.access(RCLONE_CONFIG_DIR);
+		await fs.access(RCLONE_CONFIG_FILE);
-
-		// Make sure the folder is not empty
-		const files = await fs.readdir(RCLONE_CONFIG_DIR);
-		if (files.length === 0) {
-			throw new Error("rclone config directory is empty");
-		}
 
 		logger.info("rclone capability: enabled");
 		return true;

app/server/core/constants.ts

@@ -1,3 +1,5 @@
+import path from "node:path";
+
 export const OPERATION_TIMEOUT = 5000;
 
 export const VOLUME_MOUNT_BASE = process.env.ZEROBYTE_VOLUMES_DIR || "/var/lib/zerobyte/volumes";
@@ -10,6 +12,7 @@ export const RESTIC_PASS_FILE = process.env.RESTIC_PASS_FILE || "/var/lib/zeroby
 export const SSH_KEYS_DIR = "/var/lib/zerobyte/ssh";
 
 export const RCLONE_CONFIG_DIR = process.env.RCLONE_CONFIG_DIR || "/root/.config/rclone";
+export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf");
 export const RESTORE_BLOCKED_ROOTS = [REPOSITORY_BASE, RESTIC_CACHE_DIR, SSH_KEYS_DIR, RCLONE_CONFIG_DIR, "/app"];
 
 export const DEFAULT_EXCLUDES = [RESTIC_PASS_FILE, REPOSITORY_BASE];

app/server/core/restic.ts

@@ -1,6 +1,6 @@
 import { createRestic } from "@zerobyte/core/restic/server";
 import type { ResticDeps } from "@zerobyte/core/restic";
-import { DEFAULT_EXCLUDES, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "./constants";
+import { DEFAULT_EXCLUDES, RCLONE_CONFIG_FILE, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "./constants";
 import { config } from "./config";
 import { cryptoUtils } from "../utils/crypto";
 import { db } from "../db/db";
@@ -26,6 +26,7 @@ export const resticDeps: ResticDeps = {
 	resticCacheDir: RESTIC_CACHE_DIR,
 	resticPassFile: RESTIC_PASS_FILE,
 	defaultExcludes: DEFAULT_EXCLUDES,
+	rcloneConfigFile: RCLONE_CONFIG_FILE,
 	hostname: config.resticHostname,
 };
 

app/server/modules/agents/__tests__/session.test.ts

@@ -131,6 +131,7 @@ test("close emits a synthetic backup.cancelled for a queued backup", () => {
 				cacheDir: "/tmp/cache",
 				passFile: "/tmp/pass",
 				defaultExcludes: [],
+				rcloneConfigFile: "/tmp/rclone.conf",
 			},
 		}),
 	);

app/server/modules/backends/rclone/rclone-backend.ts

@@ -1,6 +1,6 @@
 import * as fs from "node:fs/promises";
 import * as os from "node:os";
-import { OPERATION_TIMEOUT } from "../../../core/constants";
+import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE } from "../../../core/constants";
 import { toMessage } from "../../../utils/errors";
 import { logger } from "@zerobyte/core/node";
 import { getMountForPath } from "../../../utils/mountinfo";
@@ -51,7 +51,12 @@ const mount = async (config: BackendConfig, path: string) => {
 		logger.debug(`Mounting rclone volume ${path}...`);
 		logger.info(`Executing rclone: rclone ${args.join(" ")}`);
 
-		const result = await safeExec({ command: "rclone", args, timeout: zbConfig.serverIdleTimeout * 1000 });
+		const result = await safeExec({
+			command: "rclone",
+			args,
+			env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
+			timeout: zbConfig.serverIdleTimeout * 1000,
+		});
 
 		if (result.exitCode !== 0) {
 			const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";

app/server/utils/__tests__/rclone.test.ts

@@ -0,0 +1,58 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+vi.mock("@zerobyte/core/node", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("@zerobyte/core/node")>();
+
+	return {
+		...actual,
+		safeExec: vi.fn(),
+	};
+});
+
+import { safeExec } from "@zerobyte/core/node";
+import { RCLONE_CONFIG_FILE } from "../../core/constants";
+import { getRcloneRemoteInfo, listRcloneRemotes } from "../rclone";
+
+afterEach(() => {
+	vi.restoreAllMocks();
+});
+
+describe("rclone utilities", () => {
+	test("lists remotes using the configured rclone config file", async () => {
+		vi.mocked(safeExec).mockResolvedValueOnce({
+			exitCode: 0,
+			stdout: "primary:\narchive:\n",
+			stderr: "",
+			timedOut: false,
+		});
+
+		expect(await listRcloneRemotes()).toEqual(["primary", "archive"]);
+		expect(safeExec).toHaveBeenCalledWith({
+			command: "rclone",
+			args: ["listremotes"],
+			env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
+		});
+	});
+
+	test("loads remote details using the configured rclone config file", async () => {
+		vi.mocked(safeExec).mockResolvedValueOnce({
+			exitCode: 0,
+			stdout: "[primary]\ntype = s3\nprovider = AWS\n",
+			stderr: "",
+			timedOut: false,
+		});
+
+		expect(await getRcloneRemoteInfo("primary")).toEqual({
+			type: "s3",
+			config: {
+				type: "s3",
+				provider: "AWS",
+			},
+		});
+		expect(safeExec).toHaveBeenCalledWith({
+			command: "rclone",
+			args: ["config", "show", "primary"],
+			env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
+		});
+	});
+});

app/server/utils/rclone.ts

@@ -1,13 +1,18 @@
 import { logger } from "@zerobyte/core/node";
 import { toMessage } from "./errors";
 import { safeExec } from "@zerobyte/core/node";
+import { RCLONE_CONFIG_FILE } from "../core/constants";
+
+const rcloneEnv = {
+	RCLONE_CONFIG: RCLONE_CONFIG_FILE,
+};
 
 /**
  * List all configured rclone remotes
  * @returns Array of remote names
  */
 export async function listRcloneRemotes(): Promise<string[]> {
-	const result = await safeExec({ command: "rclone", args: ["listremotes"] });
+	const result = await safeExec({ command: "rclone", args: ["listremotes"], env: rcloneEnv });
 
 	if (result.exitCode !== 0) {
 		logger.error(`Failed to list rclone remotes: ${result.stderr.toString()}`);
@@ -34,7 +39,7 @@ export async function getRcloneRemoteInfo(
 	remote: string,
 ): Promise<{ type: string; config: Record<string, string> } | null> {
 	try {
-		const result = await safeExec({ command: "rclone", args: ["config", "show", remote] });
+		const result = await safeExec({ command: "rclone", args: ["config", "show", remote], env: rcloneEnv });
 
 		if (result.exitCode !== 0) {
 			logger.error(`Failed to get info for remote ${remote}: ${result.stderr.toString()}`);

apps/agent/src/__tests__/controller-session.test.ts

@@ -44,6 +44,7 @@ test("emits backup.failed when a backup command hits a restic error", async () =
 					cacheDir: "/tmp/restic-cache",
 					passFile: "/tmp/restic-pass",
 					defaultExcludes: [],
+					rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 				},
 			}),
 		);

apps/agent/src/commands/backup-run.ts

@@ -50,6 +50,7 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa
 					resticPassFile: payload.runtime.passFile,
 					defaultExcludes: payload.runtime.defaultExcludes,
 					hostname: payload.runtime.hostname,
+					rcloneConfigFile: payload.runtime.rcloneConfigFile,
 				};
 
 				const restic = createRestic(deps);

packages/contracts/src/agent-protocol.ts

@@ -26,6 +26,7 @@ const backupRuntimeSchema = z.object({
 	passFile: z.string(),
 	defaultExcludes: z.array(z.string()),
 	hostname: z.string().optional(),
+	rcloneConfigFile: z.string(),
 });
 
 const backupRunSchema = z.object({

packages/core/src/restic/commands/__tests__/backup.test.ts

@@ -14,6 +14,7 @@ const mockDeps: ResticDeps = {
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
 	hostname: "zerobyte",
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const VALID_SUMMARY = JSON.stringify({
@@ -61,10 +62,12 @@ type SetupOptions = {
  */
 const setup = ({ spawnResult = {}, onSpawnCall }: SetupOptions = {}) => {
 	let capturedArgs: string[] = [];
+	let capturedEnv: SafeSpawnParams["env"];
 
 	vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
 	vi.spyOn(spawnModule, "safeSpawn").mockImplementation((params: SafeSpawnParams) => {
 		capturedArgs = params.args;
+		capturedEnv = params.env;
 		return Promise.resolve(onSpawnCall?.(params)).then(() => ({
 			exitCode: 0,
 			summary: VALID_SUMMARY,
@@ -75,6 +78,7 @@ const setup = ({ spawnResult = {}, onSpawnCall }: SetupOptions = {}) => {
 
 	return {
 		getArgs: () => capturedArgs,
+		getEnv: () => capturedEnv,
 		hasFlag: (flag: string) => capturedArgs.includes(flag),
 		getOptionValues: (option: string): string[] => {
 			const values: string[] = [];
@@ -160,6 +164,25 @@ describe("backup command", () => {
 
 			expect(getOptionValues("--exclude").length).toBeGreaterThan(0);
 		});
+
+		test("passes RCLONE_CONFIG when backing up to an rclone repository", async () => {
+			const { getEnv } = setup();
+
+			await runBackup(
+				{
+					backend: "rclone",
+					remote: "remote",
+					path: "/repo",
+					isExistingRepository: true,
+					customPassword: "custom-password",
+				},
+				"/mnt/data",
+				{ organizationId: "org-1" },
+				mockDeps,
+			);
+
+			expect(getEnv()?.RCLONE_CONFIG).toBe(mockDeps.rcloneConfigFile);
+		});
 	});
 
 	describe("exit code handling", () => {

packages/core/src/restic/commands/__tests__/copy.test.ts

@@ -10,6 +10,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const sourceConfig = {

packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts

@@ -10,6 +10,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const config = {

packages/core/src/restic/commands/__tests__/dump.test.ts

@@ -11,6 +11,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const config = {

packages/core/src/restic/commands/__tests__/ls.test.ts

@@ -11,6 +11,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const config = {

packages/core/src/restic/commands/__tests__/restore.test.ts

@@ -12,6 +12,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const successfulRestoreSummary = JSON.stringify({

packages/core/src/restic/commands/__tests__/snapshots.test.ts

@@ -11,6 +11,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const config = {

packages/core/src/restic/commands/__tests__/tag-snapshots.test.ts

@@ -10,6 +10,7 @@ const mockDeps: ResticDeps = {
 	resticCacheDir: "/tmp/restic-cache",
 	resticPassFile: "/tmp/restic.pass",
 	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
 };
 
 const config = {

packages/core/src/restic/helpers/__tests__/build-env.test.ts

@@ -12,6 +12,7 @@ const makeDeps = (overrides: Partial<ResticDeps> = {}): ResticDeps => ({
 	resticCacheDir: RESTIC_CACHE_DIR,
 	resticPassFile: RESTIC_PASS_FILE,
 	defaultExcludes: [RESTIC_PASS_FILE, "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/tmp/rclone.conf",
 	...overrides,
 });
 

packages/core/src/restic/helpers/build-env.ts

@@ -69,6 +69,9 @@ export const buildEnv = async (
 			}
 			break;
 		}
+		case "rclone":
+			env.RCLONE_CONFIG = deps.rcloneConfigFile;
+			break;
 		case "sftp": {
 			const decryptedKey = await deps.resolveSecret(config.privateKey);
 			const keyPath = path.join("/tmp", `zerobyte-ssh-${crypto.randomBytes(8).toString("hex")}`);

packages/core/src/restic/types.ts

@@ -7,6 +7,7 @@ export interface ResticDeps {
 	resticCacheDir: string;
 	resticPassFile: string;
 	defaultExcludes: string[];
+	rcloneConfigFile: string;
 	hostname?: string;
 }