fix: restart agent after an unexpected exit
This commit is contained in:
parent
9fba2d083c
commit
4a1b479cf2
4 changed files with 109 additions and 1 deletions
82
app/server/modules/agents/__tests__/agents-manager.test.ts
Normal file
82
app/server/modules/agents/__tests__/agents-manager.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import * as childProcess from "node:child_process";
|
||||||
|
import { PassThrough } from "node:stream";
|
||||||
|
import { afterEach, expect, mock, test, vi } from "bun:test";
|
||||||
|
|
||||||
|
const spawnMock = mock();
|
||||||
|
|
||||||
|
await mock.module("node:child_process", () => ({
|
||||||
|
...childProcess,
|
||||||
|
spawn: spawnMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { spawnLocalAgent, stopLocalAgent } from "../agents-manager";
|
||||||
|
|
||||||
|
const flushMicrotasks = async () => {
|
||||||
|
await Promise.resolve();
|
||||||
|
};
|
||||||
|
|
||||||
|
type FakeChildProcess = EventEmitter & {
|
||||||
|
stdout: PassThrough;
|
||||||
|
stderr: PassThrough;
|
||||||
|
exitCode: number | null;
|
||||||
|
signalCode: NodeJS.Signals | null;
|
||||||
|
kill: ReturnType<typeof mock>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createFakeChild = () => {
|
||||||
|
const child = new EventEmitter() as FakeChildProcess;
|
||||||
|
|
||||||
|
child.stdout = new PassThrough();
|
||||||
|
child.stderr = new PassThrough();
|
||||||
|
child.exitCode = null;
|
||||||
|
child.signalCode = null;
|
||||||
|
child.kill = mock(() => {
|
||||||
|
child.exitCode = 0;
|
||||||
|
child.emit("exit", 0, null);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
return child;
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await stopLocalAgent();
|
||||||
|
spawnMock.mockReset();
|
||||||
|
mock.restore();
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("respawns the local agent after an unexpected exit", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
const firstChild = createFakeChild();
|
||||||
|
const secondChild = createFakeChild();
|
||||||
|
spawnMock.mockReturnValueOnce(firstChild).mockReturnValueOnce(secondChild);
|
||||||
|
|
||||||
|
await spawnLocalAgent();
|
||||||
|
|
||||||
|
firstChild.exitCode = 1;
|
||||||
|
firstChild.emit("exit", 1, null);
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(1_000);
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(spawnMock).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not respawn the local agent after an intentional stop", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
const child = createFakeChild();
|
||||||
|
spawnMock.mockReturnValue(child);
|
||||||
|
|
||||||
|
await spawnLocalAgent();
|
||||||
|
await stopLocalAgent();
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(1_000);
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(child.kill).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
@ -43,6 +43,8 @@ type AgentManagerRuntime = ReturnType<typeof createAgentManagerRuntime>;
|
||||||
type AgentRuntimeState = {
|
type AgentRuntimeState = {
|
||||||
agentManager: AgentManagerRuntime;
|
agentManager: AgentManagerRuntime;
|
||||||
localAgent: ChildProcess | null;
|
localAgent: ChildProcess | null;
|
||||||
|
isStoppingLocalAgent: boolean;
|
||||||
|
localAgentRestartTimeout: ReturnType<typeof setTimeout> | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
type ProcessWithAgentRuntime = NodeJS.Process & {
|
type ProcessWithAgentRuntime = NodeJS.Process & {
|
||||||
|
|
@ -60,6 +62,8 @@ const getAgentRuntimeState = () => {
|
||||||
const runtime = {
|
const runtime = {
|
||||||
agentManager: createAgentManagerRuntime(),
|
agentManager: createAgentManagerRuntime(),
|
||||||
localAgent: null,
|
localAgent: null,
|
||||||
|
isStoppingLocalAgent: false,
|
||||||
|
localAgentRestartTimeout: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
runtimeProcess.__zerobyteAgentRuntime = runtime;
|
runtimeProcess.__zerobyteAgentRuntime = runtime;
|
||||||
|
|
@ -105,28 +109,48 @@ export const spawnLocalAgent = async () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
agentProcess.on("exit", (code, signal) => {
|
agentProcess.on("exit", (code, signal) => {
|
||||||
|
const shouldRestart = runtime.localAgent === agentProcess && !runtime.isStoppingLocalAgent;
|
||||||
if (runtime.localAgent === agentProcess) {
|
if (runtime.localAgent === agentProcess) {
|
||||||
runtime.localAgent = null;
|
runtime.localAgent = null;
|
||||||
}
|
}
|
||||||
logger.info(`Agent process exited with code ${code} and signal ${signal}`);
|
logger.info(`Agent process exited with code ${code} and signal ${signal}`);
|
||||||
|
|
||||||
|
if (!shouldRestart) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
runtime.localAgentRestartTimeout = setTimeout(() => {
|
||||||
|
runtime.localAgentRestartTimeout = null;
|
||||||
|
void spawnLocalAgent().catch((error) => {
|
||||||
|
logger.error(`Failed to restart local agent: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
|
});
|
||||||
|
}, 1_000);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
export const stopLocalAgent = async () => {
|
export const stopLocalAgent = async () => {
|
||||||
const runtime = getAgentRuntimeState();
|
const runtime = getAgentRuntimeState();
|
||||||
|
if (runtime.localAgentRestartTimeout) {
|
||||||
|
clearTimeout(runtime.localAgentRestartTimeout);
|
||||||
|
runtime.localAgentRestartTimeout = null;
|
||||||
|
}
|
||||||
|
|
||||||
if (!runtime.localAgent) {
|
if (!runtime.localAgent) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const agentProcess = runtime.localAgent;
|
const agentProcess = runtime.localAgent;
|
||||||
runtime.localAgent = null;
|
runtime.localAgent = null;
|
||||||
|
runtime.isStoppingLocalAgent = true;
|
||||||
|
|
||||||
if (agentProcess.exitCode !== null || agentProcess.signalCode !== null) {
|
if (agentProcess.exitCode !== null || agentProcess.signalCode !== null) {
|
||||||
|
runtime.isStoppingLocalAgent = false;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const exited = new Promise<void>((resolve) => {
|
const exited = new Promise<void>((resolve) => {
|
||||||
agentProcess.once("exit", () => {
|
agentProcess.once("exit", () => {
|
||||||
|
runtime.isStoppingLocalAgent = false;
|
||||||
resolve();
|
resolve();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,12 @@
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"plugins": [{ "name": "@effect/language-service" }],
|
"plugins": [{ "name": "@effect/language-service" }],
|
||||||
// Environment setup & latest features
|
// Environment setup & latest features
|
||||||
"lib": ["ESNext"],
|
"lib": ["DOM", "ESNext"],
|
||||||
"target": "ESNext",
|
"target": "ESNext",
|
||||||
"module": "Preserve",
|
"module": "Preserve",
|
||||||
"moduleDetection": "force",
|
"moduleDetection": "force",
|
||||||
"jsx": "react-jsx",
|
"jsx": "react-jsx",
|
||||||
|
"types": ["bun", "node"],
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
|
|
||||||
// Bundler mode
|
// Bundler mode
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
"module": "Preserve",
|
"module": "Preserve",
|
||||||
"moduleDetection": "force",
|
"moduleDetection": "force",
|
||||||
"jsx": "react-jsx",
|
"jsx": "react-jsx",
|
||||||
|
"types": ["bun", "node"],
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
|
|
||||||
// Bundler mode
|
// Bundler mode
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue