74a05a8de0
Move orphaned operation recovery logic into a dedicated orphan-recovery module, introducing a periodic sweep timer that triggers recovery every 15 seconds while the worker is connected. Refactor runtime to delegate orphan detection and handling to the new module, improving modularity and maintainability. Add unit tests for orphan-recovery to ensure correctness.
59 lines
1.7 KiB
TypeScript
59 lines
1.7 KiB
TypeScript
import { afterEach, describe, expect, test, vi } from 'vitest';
|
|
import { recoverOrphanedOperations } from '../../src/orphan-recovery';
|
|
import { FakeControlPlane } from '../fixtures/fake-control-plane';
|
|
|
|
describe('orphan recovery', () => {
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
test('recovers a running whisper op when a later sweep crosses the timeout in the same session', async () => {
|
|
vi.useFakeTimers();
|
|
vi.setSystemTime(new Date('2026-06-03T08:00:00.000Z'));
|
|
|
|
const fake = new FakeControlPlane();
|
|
const startedAt = Date.now();
|
|
fake.seedState({
|
|
opId: 'op-whisper-running',
|
|
opKey: 'k-whisper-running',
|
|
kind: 'whisper_align',
|
|
jobId: 'job-op-whisper-running',
|
|
status: 'running',
|
|
queuedAt: startedAt,
|
|
updatedAt: startedAt,
|
|
});
|
|
|
|
const firstSweep = await recoverOrphanedOperations({
|
|
operationStateStore: fake.deps.operationStateStore,
|
|
orchestrator: fake.deps.orchestrator,
|
|
whisperTimeoutMs: 30_000,
|
|
pdfTimeoutMs: 300_000,
|
|
opStaleMs: 1_800_000,
|
|
});
|
|
expect(firstSweep).toEqual([]);
|
|
expect(fake.getState('op-whisper-running')).toMatchObject({
|
|
status: 'running',
|
|
});
|
|
|
|
vi.advanceTimersByTime(31_000);
|
|
|
|
const secondSweep = await recoverOrphanedOperations({
|
|
operationStateStore: fake.deps.operationStateStore,
|
|
orchestrator: fake.deps.orchestrator,
|
|
whisperTimeoutMs: 30_000,
|
|
pdfTimeoutMs: 300_000,
|
|
opStaleMs: 1_800_000,
|
|
});
|
|
expect(secondSweep).toEqual([{
|
|
opId: 'op-whisper-running',
|
|
kind: 'whisper_align',
|
|
status: 'running',
|
|
}]);
|
|
expect(fake.getState('op-whisper-running')).toMatchObject({
|
|
status: 'failed',
|
|
error: {
|
|
code: 'WORKER_ORPHANED_OP',
|
|
},
|
|
});
|
|
});
|
|
});
|