refactor: use spawn & exec instead of bun $ (#356)

* refactor: use spawn & exec accordingly

* chore: pr feedbacks
This commit is contained in:
Nico 2026-01-15 22:09:00 +01:00 committed by GitHub
parent 2cbd0d1021
commit d3f9ced5dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 108 additions and 109 deletions

View file

@ -1,6 +1,5 @@
import * as fs from "node:fs/promises"; import * as fs from "node:fs/promises";
import * as os from "node:os"; import * as os from "node:os";
import { $ } from "bun";
import { OPERATION_TIMEOUT } from "../../../core/constants"; import { OPERATION_TIMEOUT } from "../../../core/constants";
import { toMessage } from "../../../utils/errors"; import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger"; import { logger } from "../../../utils/logger";
@ -9,6 +8,7 @@ import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend"; import type { VolumeBackend } from "../backend";
import { executeUnmount } from "../utils/backend-utils"; import { executeUnmount } from "../utils/backend-utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes"; import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
import { exec } from "~/server/utils/spawn";
const mount = async (config: BackendConfig, path: string) => { const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting rclone volume ${path}...`); logger.debug(`Mounting rclone volume ${path}...`);
@ -50,7 +50,7 @@ const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting rclone volume ${path}...`); logger.debug(`Mounting rclone volume ${path}...`);
logger.info(`Executing rclone: rclone ${args.join(" ")}`); logger.info(`Executing rclone: rclone ${args.join(" ")}`);
const result = await $`rclone ${args}`.nothrow(); const result = await exec({ command: "rclone", args });
if (result.exitCode !== 0) { if (result.exitCode !== 0) {
const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error"; const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";

View file

@ -1,8 +1,8 @@
import * as fs from "node:fs/promises"; import * as fs from "node:fs/promises";
import * as npath from "node:path"; import * as npath from "node:path";
import { $ } from "bun";
import { toMessage } from "../../../utils/errors"; import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger"; import { logger } from "../../../utils/logger";
import { exec } from "~/server/utils/spawn";
export const executeMount = async (args: string[]): Promise<void> => { export const executeMount = async (args: string[]): Promise<void> => {
const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production"; const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production";
@ -10,7 +10,7 @@ export const executeMount = async (args: string[]): Promise<void> => {
const effectiveArgs = shouldBeVerbose && !hasVerboseFlag ? ["-v", ...args] : args; const effectiveArgs = shouldBeVerbose && !hasVerboseFlag ? ["-v", ...args] : args;
logger.debug(`Executing mount ${effectiveArgs.join(" ")}`); logger.debug(`Executing mount ${effectiveArgs.join(" ")}`);
let result = await $`mount ${effectiveArgs}`.nothrow(); const result = await exec({ command: "mount", args: effectiveArgs, timeout: 10000 });
const stdout = result.stdout.toString().trim(); const stdout = result.stdout.toString().trim();
const stderr = result.stderr.toString().trim(); const stderr = result.stderr.toString().trim();
@ -31,7 +31,8 @@ export const executeUnmount = async (path: string): Promise<void> => {
let stderr: string | undefined; let stderr: string | undefined;
logger.debug(`Executing umount -l ${path}`); logger.debug(`Executing umount -l ${path}`);
const result = await $`umount -l ${path}`.nothrow(); const result = await exec({ command: "umount", args: ["-l", path], timeout: 10000 });
stderr = result.stderr.toString(); stderr = result.stderr.toString();
if (stderr?.trim()) { if (stderr?.trim()) {

View file

@ -7,7 +7,7 @@ import { generateBackupOutput } from "~/test/helpers/restic";
import { faker } from "@faker-js/faker"; import { faker } from "@faker-js/faker";
import * as spawnModule from "~/server/utils/spawn"; import * as spawnModule from "~/server/utils/spawn";
const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, stdout: "", stderr: "" })); const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: "", error: "" }));
beforeEach(() => { beforeEach(() => {
resticBackupMock.mockClear(); resticBackupMock.mockClear();
@ -31,7 +31,7 @@ describe("execute backup", () => {
expect(schedule.nextBackupAt).toBeNull(); expect(schedule.nextBackupAt).toBeNull();
resticBackupMock.mockImplementationOnce(() => resticBackupMock.mockImplementationOnce(() =>
Promise.resolve({ exitCode: 0, stdout: generateBackupOutput(), stderr: "" }), Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }),
); );
// act // act
@ -76,7 +76,7 @@ describe("execute backup", () => {
}); });
resticBackupMock.mockImplementationOnce(() => resticBackupMock.mockImplementationOnce(() =>
Promise.resolve({ exitCode: 0, stdout: generateBackupOutput(), stderr: "" }), Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }),
); );
// act // act
@ -97,7 +97,7 @@ describe("execute backup", () => {
resticBackupMock.mockImplementation(async () => { resticBackupMock.mockImplementation(async () => {
await new Promise((resolve) => setTimeout(resolve, 100)); await new Promise((resolve) => setTimeout(resolve, 100));
return Promise.resolve({ exitCode: 0, stdout: generateBackupOutput(), stderr: "" }); return Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" });
}); });
// act // act
@ -119,7 +119,7 @@ describe("execute backup", () => {
}); });
resticBackupMock.mockImplementationOnce(() => resticBackupMock.mockImplementationOnce(() =>
Promise.resolve({ exitCode: 3, stdout: generateBackupOutput(), stderr: "Some error occurred" }), Promise.resolve({ exitCode: 3, summary: generateBackupOutput(), error: "Some error occurred" }),
); );
// act // act
@ -140,7 +140,7 @@ describe("execute backup", () => {
}); });
resticBackupMock.mockImplementationOnce(() => resticBackupMock.mockImplementationOnce(() =>
Promise.resolve({ exitCode: 1, stdout: generateBackupOutput(), stderr: "Some error occurred" }), Promise.resolve({ exitCode: 1, summary: generateBackupOutput(), error: "Some error occurred" }),
); );
// act // act

View file

@ -3,7 +3,7 @@ import { db } from "../../../db/db";
import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../../db/schema"; import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../../db/schema";
import { logger } from "../../../utils/logger"; import { logger } from "../../../utils/logger";
import { toMessage } from "~/server/utils/errors"; import { toMessage } from "~/server/utils/errors";
import { safeSpawn } from "~/server/utils/spawn"; import { exec } from "~/server/utils/spawn";
import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "~/server/utils/restic"; import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "~/server/utils/restic";
const migrateTag = async ( const migrateTag = async (
@ -20,7 +20,7 @@ const migrateTag = async (
addCommonArgs(args, env); addCommonArgs(args, env);
logger.info(`Migrating snapshots for schedule '${scheduleName}' from tag '${oldTag}' to '${newTag}'`); logger.info(`Migrating snapshots for schedule '${scheduleName}' from tag '${oldTag}' to '${newTag}'`);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {

View file

@ -1,13 +1,13 @@
import { $ } from "bun";
import { logger } from "./logger"; import { logger } from "./logger";
import { toMessage } from "./errors"; import { toMessage } from "./errors";
import { exec } from "./spawn";
/** /**
* List all configured rclone remotes * List all configured rclone remotes
* @returns Array of remote names * @returns Array of remote names
*/ */
export async function listRcloneRemotes(): Promise<string[]> { export async function listRcloneRemotes(): Promise<string[]> {
const result = await $`rclone listremotes`.nothrow(); const result = await exec({ command: "rclone", args: ["listremotes"] });
if (result.exitCode !== 0) { if (result.exitCode !== 0) {
logger.error(`Failed to list rclone remotes: ${result.stderr.toString()}`); logger.error(`Failed to list rclone remotes: ${result.stderr.toString()}`);
@ -34,7 +34,7 @@ export async function getRcloneRemoteInfo(
remote: string, remote: string,
): Promise<{ type: string; config: Record<string, string> } | null> { ): Promise<{ type: string; config: Record<string, string> } | null> {
try { try {
const result = await $`rclone config show ${remote}`.quiet(); const result = await exec({ command: "rclone", args: ["config", "show", remote] });
if (result.exitCode !== 0) { if (result.exitCode !== 0) {
logger.error(`Failed to get info for remote ${remote}: ${result.stderr.toString()}`); logger.error(`Failed to get info for remote ${remote}: ${result.stderr.toString()}`);

View file

@ -9,7 +9,7 @@ import { config as appConfig } from "../core/config";
import { logger } from "./logger"; import { logger } from "./logger";
import { cryptoUtils } from "./crypto"; import { cryptoUtils } from "./crypto";
import type { RetentionPolicy } from "../modules/backups/backups.dto"; import type { RetentionPolicy } from "../modules/backups/backups.dto";
import { safeSpawn } from "./spawn"; import { safeSpawn, exec } from "./spawn";
import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic"; import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic";
import { ResticError } from "./errors"; import { ResticError } from "./errors";
@ -231,7 +231,7 @@ const init = async (config: RepositoryConfig) => {
const args = ["init", "--repo", repoUrl]; const args = ["init", "--repo", repoUrl];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -341,8 +341,6 @@ const backup = async (
} }
}, 1000); }, 1000);
let stdout = "";
logger.debug(`Executing: restic ${args.join(" ")}`); logger.debug(`Executing: restic ${args.join(" ")}`);
const res = await safeSpawn({ const res = await safeSpawn({
command: "restic", command: "restic",
@ -350,37 +348,34 @@ const backup = async (
env, env,
signal: options?.signal, signal: options?.signal,
onStdout: (data) => { onStdout: (data) => {
stdout = data;
logData(data); logData(data);
if (options?.onProgress) { if (options?.onProgress) {
streamProgress(data); streamProgress(data);
} }
}, },
finally: async () => {
if (includeFile) await fs.unlink(includeFile).catch(() => {});
if (excludeFile) await fs.unlink(excludeFile).catch(() => {});
await cleanupTemporaryKeys(env);
},
}); });
if (includeFile) await fs.unlink(includeFile).catch(() => {});
if (excludeFile) await fs.unlink(excludeFile).catch(() => {});
await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) { if (options?.signal?.aborted) {
logger.warn("Restic backup was aborted by signal."); logger.warn("Restic backup was aborted by signal.");
return { result: null, exitCode: res.exitCode }; return { result: null, exitCode: res.exitCode };
} }
if (res.exitCode === 3) { if (res.exitCode === 3) {
logger.error(`Restic backup encountered read errors: ${res.stderr}`); logger.error(`Restic backup encountered read errors: ${res.error}`);
} }
if (res.exitCode !== 0 && res.exitCode !== 3) { if (res.exitCode !== 0 && res.exitCode !== 3) {
logger.error(`Restic backup failed: ${res.stderr}`); logger.error(`Restic backup failed: ${res.error}`);
logger.error(`Command executed: restic ${args.join(" ")}`); logger.error(`Command executed: restic ${args.join(" ")}`);
throw new ResticError(res.exitCode, res.stderr); throw new ResticError(res.exitCode, res.error);
} }
const lastLine = (stdout || res.stdout).trim(); const lastLine = res.summary.trim();
let summaryLine = ""; let summaryLine = "";
try { try {
const resSummary = JSON.parse(lastLine ?? "{}"); const resSummary = JSON.parse(lastLine ?? "{}");
@ -390,6 +385,7 @@ const backup = async (
summaryLine = "{}"; summaryLine = "{}";
} }
logger.debug(`Restic backup output last line: ${summaryLine}`);
const result = backupOutputSchema(summaryLine); const result = backupOutputSchema(summaryLine);
if (result instanceof type.errors) { if (result instanceof type.errors) {
@ -461,27 +457,22 @@ const restore = async (
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
logger.error(`Restic restore failed: ${res.stderr}`); logger.error(`Restic restore failed: ${res.error}`);
throw new ResticError(res.exitCode, res.stderr); throw new ResticError(res.exitCode, res.error);
} }
const outputLines = res.stdout.trim().split("\n"); const lastLine = res.summary.trim();
const lastLine = outputLines[outputLines.length - 1]; let summaryLine = "";
try {
if (!lastLine) { const resSummary = JSON.parse(lastLine ?? "{}");
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`); summaryLine = resSummary;
return { } catch (_) {
message_type: "summary" as const, logger.warn("Failed to parse restic backup output JSON summary.", lastLine);
total_files: 0, summaryLine = "{}";
files_restored: 0,
files_skipped: 0,
bytes_skipped: 0,
};
} }
logger.debug(`Restic restore output last line: ${lastLine}`); logger.debug(`Restic restore output last line: ${summaryLine}`);
const resSummary = JSON.parse(lastLine); const result = restoreOutputSchema(summaryLine);
const result = restoreOutputSchema(resSummary);
if (result instanceof type.errors) { if (result instanceof type.errors) {
logger.warn(`Restic restore output validation failed: ${result.summary}`); logger.warn(`Restic restore output validation failed: ${result.summary}`);
@ -518,7 +509,7 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[] }
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -567,7 +558,7 @@ const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra:
args.push("--prune"); args.push("--prune");
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -589,7 +580,7 @@ const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[])
const args: string[] = ["--repo", repoUrl, "forget", ...snapshotIds, "--prune"]; const args: string[] = ["--repo", repoUrl, "forget", ...snapshotIds, "--prune"];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -638,7 +629,7 @@ const tagSnapshots = async (
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -688,7 +679,7 @@ const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) =
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -739,7 +730,7 @@ const unlock = async (config: RepositoryConfig) => {
const args = ["unlock", "--repo", repoUrl, "--remove-all"]; const args = ["unlock", "--repo", repoUrl, "--remove-all"];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -763,7 +754,7 @@ const check = async (config: RepositoryConfig, options?: { readData?: boolean })
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
const { stdout, stderr } = res; const { stdout, stderr } = res;
@ -796,7 +787,7 @@ const repairIndex = async (config: RepositoryConfig) => {
const args = ["repair", "index", "--repo", repoUrl]; const args = ["repair", "index", "--repo", repoUrl];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
const { stdout, stderr } = res; const { stdout, stderr } = res;
@ -863,7 +854,7 @@ const copy = async (
logger.info(`Copying snapshots from ${sourceRepoUrl} to ${destRepoUrl}...`); logger.info(`Copying snapshots from ${sourceRepoUrl} to ${destRepoUrl}...`);
logger.debug(`Executing: restic ${args.join(" ")}`); logger.debug(`Executing: restic ${args.join(" ")}`);
const res = await safeSpawn({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(sourceEnv); await cleanupTemporaryKeys(sourceEnv);
await cleanupTemporaryKeys(destEnv); await cleanupTemporaryKeys(destEnv);

View file

@ -1,4 +1,4 @@
import { safeSpawn } from "./spawn"; import { exec } from "./spawn";
import { logger } from "./logger"; import { logger } from "./logger";
import { toMessage } from "./errors"; import { toMessage } from "./errors";
@ -16,10 +16,7 @@ export async function sendNotification(params: SendNotificationParams) {
logger.debug(`Sending notification via Shoutrrr: ${title}`); logger.debug(`Sending notification via Shoutrrr: ${title}`);
const result = await safeSpawn({ const result = await exec({ command: "shoutrrr", args });
command: "shoutrrr",
args,
});
if (result.exitCode === 0) { if (result.exitCode === 0) {
logger.debug(`Notification sent successfully: ${title}`); logger.debug(`Notification sent successfully: ${title}`);

View file

@ -1,78 +1,88 @@
import { spawn } from "node:child_process"; import { spawn, execFile, type ExecException, type ExecFileOptions } from "node:child_process";
import { createInterface } from "node:readline";
import { promisify } from "node:util";
type ExecProps = {
command: string;
args?: string[];
env?: NodeJS.ProcessEnv;
} & ExecFileOptions;
export const exec = async ({ command, args = [], env = {}, ...rest }: ExecProps) => {
const options = {
env: { ...process.env, ...env },
};
try {
const { stdout, stderr } = await promisify(execFile)(command, args, { ...options, ...rest, encoding: "utf8" });
return { exitCode: 0, stdout, stderr };
} catch (error) {
const execError = error as ExecException;
return {
exitCode: typeof execError.code === "number" ? execError.code : 1,
stdout: execError.stdout || "",
stderr: execError.stderr || "",
};
}
};
export interface SafeSpawnParams { export interface SafeSpawnParams {
command: string; command: string;
args: string[]; args: string[];
env?: NodeJS.ProcessEnv; env?: NodeJS.ProcessEnv;
signal?: AbortSignal; signal?: AbortSignal;
onStdout?: (data: string) => void; onStdout?: (line: string) => void;
onStderr?: (error: string) => void; onStderr?: (error: string) => void;
onError?: (error: Error) => Promise<void> | void;
onClose?: (code: number | null) => Promise<void> | void;
finally?: () => Promise<void> | void;
} }
type SpawnResult = { type SpawnResult = {
exitCode: number; exitCode: number;
stdout: string; summary: string;
stderr: string; error: string;
}; };
export const safeSpawn = (params: SafeSpawnParams) => { export const safeSpawn = (params: SafeSpawnParams) => {
const { command, args, env = {}, signal, ...callbacks } = params; const { command, args, env = {}, signal, onStdout, onStderr } = params;
let lastStdout = "";
let lastStderr = "";
return new Promise<SpawnResult>((resolve) => { return new Promise<SpawnResult>((resolve) => {
let stdoutData = "";
let stderrData = "";
const child = spawn(command, args, { const child = spawn(command, args, {
env: { ...process.env, ...env }, env: { ...process.env, ...env },
signal: signal, signal: signal,
}); });
child.stdout.on("data", (data) => { child.stdout.setEncoding("utf8");
if (callbacks.onStdout) { child.stderr.setEncoding("utf8");
callbacks.onStdout(data.toString());
} else { const rl = createInterface({ input: child.stdout });
stdoutData += data.toString(); const rlErr = createInterface({ input: child.stderr });
rl.on("line", (line) => {
if (onStdout) onStdout(line);
const trimmed = line.trim();
if (trimmed.length > 0) {
lastStdout = line;
} }
}); });
child.stderr.on("data", (data) => { rlErr.on("line", (line) => {
if (callbacks.onStderr) { if (onStderr) onStderr(line);
callbacks.onStderr(data.toString()); const trimmed = line.trim();
if (trimmed.length > 0) {
lastStderr = line;
} }
stderrData += data.toString();
}); });
child.on("error", async (error) => { child.on("error", (err) => {
if (callbacks.onError) { resolve({ exitCode: -1, summary: lastStdout, error: err.message || lastStderr });
await callbacks.onError(error);
}
if (callbacks.finally) {
await callbacks.finally();
}
resolve({
exitCode: -1,
stdout: stdoutData,
stderr: stderrData,
});
}); });
child.on("close", async (code) => { child.on("close", (code) => {
if (callbacks.onClose) { resolve({ exitCode: code ?? -1, summary: lastStdout, error: lastStderr });
await callbacks.onClose(code);
}
if (callbacks.finally) {
await callbacks.finally();
}
resolve({
exitCode: code === null ? -1 : code,
stdout: stdoutData,
stderr: stderrData,
});
}); });
}); });
}; };

View file

@ -14,7 +14,7 @@ export default defineConfig({
}, },
}, },
server: { server: {
host: true, host: '0.0.0.0',
port: 4096, port: 4096,
}, },
}); });