fix(snapshots): use streamed response to avoid 1 MiB buffer limit on large repos (#771)

packages/core/src/restic/commands/__tests__/snapshots.test.ts

@@ -0,0 +1,63 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import * as cleanupModule from "../../helpers/cleanup-temporary-keys";
+import * as nodeModule from "../../../node";
+import { snapshots } from "../snapshots";
+import type { SafeSpawnParams } from "../../../node";
+import type { ResticDeps } from "../../types";
+
+const mockDeps: ResticDeps = {
+	resolveSecret: async (s) => s,
+	getOrganizationResticPassword: async () => "org-restic-password",
+	resticCacheDir: "/tmp/restic-cache",
+	resticPassFile: "/tmp/restic.pass",
+	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+};
+
+const config = {
+	backend: "local" as const,
+	path: "/tmp/restic-repo",
+	isExistingRepository: true,
+	customPassword: "custom-password",
+};
+
+const getOnStdout = (params: SafeSpawnParams) => {
+	if (params.stdoutMode === "raw") {
+		return undefined;
+	}
+
+	return params.onStdout;
+};
+
+afterEach(() => {
+	vi.restoreAllMocks();
+});
+
+describe("snapshots command", () => {
+	test("parses a streamed multi-line JSON array", async () => {
+		const snapshotsOutput = [
+			{
+				hostname: "host",
+				id: "snapshot-1",
+				paths: ["/data"],
+				short_id: "snapshot-1",
+				tags: ["daily"],
+				time: "2025-01-01T00:00:00Z",
+			},
+		];
+
+		vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
+		vi.spyOn(nodeModule, "safeSpawn").mockImplementation((params) => {
+			const onStdout = getOnStdout(params);
+
+			for (const line of JSON.stringify(snapshotsOutput, null, 2).split("\n")) {
+				onStdout?.(line);
+			}
+
+			return Promise.resolve({ exitCode: 0, summary: "", error: "" });
+		});
+
+		const result = await snapshots(config, { organizationId: "org-1" }, mockDeps);
+
+		expect(result).toEqual(snapshotsOutput);
+	});
+});

packages/core/src/restic/commands/snapshots.ts

@@ -5,7 +5,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import { resticSnapshotSummarySchema } from "../restic-dto";
 import type { RepositoryConfig } from "../schemas";
-import { logger, safeExec } from "../../node";
+import { logger, safeSpawn } from "../../node";
 import type { ResticDeps } from "../types";
 
 const snapshotInfoSchema = z.object({
@@ -43,15 +43,24 @@ export const snapshots = async (
 
 	addCommonArgs(args, env, config);
 
-	const res = await safeExec({ command: "restic", args, env });
+	const stdoutLines: string[] = [];
+	const res = await safeSpawn({
+		command: "restic",
+		args,
+		env,
+		onStdout: (line) => {
+			stdoutLines.push(line);
+		},
+	});
 	await cleanupTemporaryKeys(env, deps);
 
 	if (res.exitCode !== 0) {
-		logger.error(`Restic snapshots retrieval failed: ${res.stderr}`);
-		throw new Error(`Restic snapshots retrieval failed: ${res.stderr}`);
+		const errorMessage = res.stderr || res.error;
+		logger.error(`Restic snapshots retrieval failed: ${errorMessage}`);
+		throw new Error(`Restic snapshots retrieval failed: ${errorMessage}`);
 	}
 
-	const result = snapshotInfoSchema.array().safeParse(JSON.parse(res.stdout));
+	const result = snapshotInfoSchema.array().safeParse(JSON.parse(stdoutLines.join("\n")));
 
 	if (!result.success) {
 		logger.error(`Restic snapshots output validation failed: ${result.error.message}`);

packages/core/src/utils/__tests__/spawn.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, test } from "vitest";
+import { safeExec } from "../spawn";
+
+describe("safeExec", () => {
+	test("falls back to the process error message when stderr is empty", async () => {
+		const result = await safeExec({
+			command: process.execPath,
+			args: ["-e", "process.stdout.write('a'.repeat(2 * 1024 * 1024))"],
+		});
+
+		expect(result.exitCode).toBe(1);
+		expect(result.stderr).toContain("stdout maxBuffer length exceeded");
+	});
+});

packages/core/src/utils/spawn.ts

@@ -28,7 +28,7 @@ export const safeExec = async ({ command, args = [], env = {}, ...rest }: ExecPr
 		return {
 			exitCode: typeof execError.code === "number" ? execError.code : 1,
 			stdout: execError.stdout || "",
-			stderr: timedOut ? "Command timed out before completing" : execError.stderr || "",
+			stderr: timedOut ? "Command timed out before completing" : execError.stderr || execError.message || "",
 			timedOut,
 		};
 	}