fix: restart agent after an unexpected exit
This commit is contained in:
parent
9fba2d083c
commit
4a1b479cf2
4 changed files with 109 additions and 1 deletions
82
app/server/modules/agents/__tests__/agents-manager.test.ts
Normal file
82
app/server/modules/agents/__tests__/agents-manager.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { EventEmitter } from "node:events";
|
||||
import * as childProcess from "node:child_process";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, expect, mock, test, vi } from "bun:test";
|
||||
|
||||
const spawnMock = mock();
|
||||
|
||||
await mock.module("node:child_process", () => ({
|
||||
...childProcess,
|
||||
spawn: spawnMock,
|
||||
}));
|
||||
|
||||
import { spawnLocalAgent, stopLocalAgent } from "../agents-manager";
|
||||
|
||||
const flushMicrotasks = async () => {
|
||||
await Promise.resolve();
|
||||
};
|
||||
|
||||
type FakeChildProcess = EventEmitter & {
|
||||
stdout: PassThrough;
|
||||
stderr: PassThrough;
|
||||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
kill: ReturnType<typeof mock>;
|
||||
};
|
||||
|
||||
const createFakeChild = () => {
|
||||
const child = new EventEmitter() as FakeChildProcess;
|
||||
|
||||
child.stdout = new PassThrough();
|
||||
child.stderr = new PassThrough();
|
||||
child.exitCode = null;
|
||||
child.signalCode = null;
|
||||
child.kill = mock(() => {
|
||||
child.exitCode = 0;
|
||||
child.emit("exit", 0, null);
|
||||
return true;
|
||||
});
|
||||
|
||||
return child;
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await stopLocalAgent();
|
||||
spawnMock.mockReset();
|
||||
mock.restore();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
test("respawns the local agent after an unexpected exit", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const firstChild = createFakeChild();
|
||||
const secondChild = createFakeChild();
|
||||
spawnMock.mockReturnValueOnce(firstChild).mockReturnValueOnce(secondChild);
|
||||
|
||||
await spawnLocalAgent();
|
||||
|
||||
firstChild.exitCode = 1;
|
||||
firstChild.emit("exit", 1, null);
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(spawnMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("does not respawn the local agent after an intentional stop", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const child = createFakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
|
||||
await spawnLocalAgent();
|
||||
await stopLocalAgent();
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
expect(child.kill).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
|
@ -43,6 +43,8 @@ type AgentManagerRuntime = ReturnType<typeof createAgentManagerRuntime>;
|
|||
type AgentRuntimeState = {
|
||||
agentManager: AgentManagerRuntime;
|
||||
localAgent: ChildProcess | null;
|
||||
isStoppingLocalAgent: boolean;
|
||||
localAgentRestartTimeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
|
||||
type ProcessWithAgentRuntime = NodeJS.Process & {
|
||||
|
|
@ -60,6 +62,8 @@ const getAgentRuntimeState = () => {
|
|||
const runtime = {
|
||||
agentManager: createAgentManagerRuntime(),
|
||||
localAgent: null,
|
||||
isStoppingLocalAgent: false,
|
||||
localAgentRestartTimeout: null,
|
||||
};
|
||||
|
||||
runtimeProcess.__zerobyteAgentRuntime = runtime;
|
||||
|
|
@ -105,28 +109,48 @@ export const spawnLocalAgent = async () => {
|
|||
});
|
||||
|
||||
agentProcess.on("exit", (code, signal) => {
|
||||
const shouldRestart = runtime.localAgent === agentProcess && !runtime.isStoppingLocalAgent;
|
||||
if (runtime.localAgent === agentProcess) {
|
||||
runtime.localAgent = null;
|
||||
}
|
||||
logger.info(`Agent process exited with code ${code} and signal ${signal}`);
|
||||
|
||||
if (!shouldRestart) {
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.localAgentRestartTimeout = setTimeout(() => {
|
||||
runtime.localAgentRestartTimeout = null;
|
||||
void spawnLocalAgent().catch((error) => {
|
||||
logger.error(`Failed to restart local agent: ${error instanceof Error ? error.message : String(error)}`);
|
||||
});
|
||||
}, 1_000);
|
||||
});
|
||||
};
|
||||
|
||||
export const stopLocalAgent = async () => {
|
||||
const runtime = getAgentRuntimeState();
|
||||
if (runtime.localAgentRestartTimeout) {
|
||||
clearTimeout(runtime.localAgentRestartTimeout);
|
||||
runtime.localAgentRestartTimeout = null;
|
||||
}
|
||||
|
||||
if (!runtime.localAgent) {
|
||||
return;
|
||||
}
|
||||
|
||||
const agentProcess = runtime.localAgent;
|
||||
runtime.localAgent = null;
|
||||
runtime.isStoppingLocalAgent = true;
|
||||
|
||||
if (agentProcess.exitCode !== null || agentProcess.signalCode !== null) {
|
||||
runtime.isStoppingLocalAgent = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const exited = new Promise<void>((resolve) => {
|
||||
agentProcess.once("exit", () => {
|
||||
runtime.isStoppingLocalAgent = false;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
"compilerOptions": {
|
||||
"plugins": [{ "name": "@effect/language-service" }],
|
||||
// Environment setup & latest features
|
||||
"lib": ["ESNext"],
|
||||
"lib": ["DOM", "ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"types": ["bun", "node"],
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
"module": "Preserve",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"types": ["bun", "node"],
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
|
|
|
|||
Loading…
Reference in a new issue