From 80739cce9409cc77b055d47886d90080a72b8d9d Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 2 Dec 2025 12:38:50 +0000 Subject: [PATCH] test: Add rescheduleTask tests for monitoring package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive tests for the rescheduleTask function covering: - Nil taskQueue handling (early return) - Successful task outcome (regular rescheduling) - Transient failure with backoff - Non-transient failure routing to dead letter queue - Exceeded retry attempts routing to dead letter queue - No outcome uses default interval - PBS and PMG instance type intervals - Adaptive polling max interval capping - Existing interval preservation Coverage: rescheduleTask 32.1% → 58.9% Coverage: monitoring package 52.8% → 53.5% --- internal/monitoring/monitor_polling_test.go | 422 ++++++++++++++++++++ 1 file changed, 422 insertions(+) diff --git a/internal/monitoring/monitor_polling_test.go b/internal/monitoring/monitor_polling_test.go index 86d4369..48ca1af 100644 --- a/internal/monitoring/monitor_polling_test.go +++ b/internal/monitoring/monitor_polling_test.go @@ -381,3 +381,425 @@ func TestDescribeInstancesForScheduler_NilSchedulerAndTracker(t *testing.T) { t.Error("expected LastSuccess to be zero with nil stalenessTracker") } } + +func TestRescheduleTask_NilTaskQueue(t *testing.T) { + m := &Monitor{ + taskQueue: nil, // nil queue + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + // Should not panic with nil taskQueue, just returns early + m.rescheduleTask(task) +} + +func TestRescheduleTask_SuccessfulOutcome(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // Record a successful outcome + m.lastOutcome[key] = taskOutcome{success: true} + + m.rescheduleTask(task) + + // Task should be rescheduled at regular interval (no backoff) + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected task to be rescheduled") + } + + // Should use base interval since scheduler is nil + if entry.task.Interval != cfg.PVEPollingInterval { + t.Errorf("expected interval %v, got %v", cfg.PVEPollingInterval, entry.task.Interval) + } +} + +func TestRescheduleTask_TransientFailureWithBackoff(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + maxRetryAttempts: 5, + backoffCfg: backoffConfig{ + Initial: 5 * time.Second, + Multiplier: 2, + Jitter: 0, // no jitter for predictable testing + Max: 5 * time.Minute, + }, + } + + // Add randomFloat method for backoff calculation + m.rng = nil // will use default random + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // Record a transient failure (1st attempt, below maxRetryAttempts) + m.failureCounts[key] = 1 + m.lastOutcome[key] = taskOutcome{ + success: false, + transient: true, + err: errors.New("connection timeout"), + } + + m.rescheduleTask(task) + + // Task should be rescheduled with backoff delay + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected task to be rescheduled with backoff") + } + + // With backoff, interval should be modified + if entry.task.Interval <= 0 { + t.Errorf("expected positive backoff interval, got %v", entry.task.Interval) + } +} + +func TestRescheduleTask_NonTransientFailureGoesToDeadLetter(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + } + + deadLetterQ := NewTaskQueue() + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + deadLetterQueue: deadLetterQ, + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + maxRetryAttempts: 5, + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // Record a non-transient failure (permanent error) + m.failureCounts[key] = 1 + m.lastOutcome[key] = taskOutcome{ + success: false, + transient: false, // non-transient + err: errors.New("authentication failed"), + } + + m.rescheduleTask(task) + + // Task should NOT be in the main queue + m.taskQueue.mu.Lock() + _, inMainQueue := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if inMainQueue { + t.Error("expected task to NOT be in main queue after non-transient failure") + } + + // Task should be in dead letter queue + deadLetterQ.mu.Lock() + dlqSize := len(deadLetterQ.entries) + deadLetterQ.mu.Unlock() + + if dlqSize != 1 { + t.Errorf("expected 1 task in dead letter queue, got %d", dlqSize) + } +} + +func TestRescheduleTask_ExceededRetryAttemptsGoesToDeadLetter(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + } + + deadLetterQ := NewTaskQueue() + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + deadLetterQueue: deadLetterQ, + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + maxRetryAttempts: 3, + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // Exceed max retry attempts (failureCount >= maxRetryAttempts) + m.failureCounts[key] = 3 + m.lastOutcome[key] = taskOutcome{ + success: false, + transient: true, // transient, but exceeded retries + err: errors.New("connection timeout"), + } + + m.rescheduleTask(task) + + // Task should be in dead letter queue + deadLetterQ.mu.Lock() + dlqSize := len(deadLetterQ.entries) + deadLetterQ.mu.Unlock() + + if dlqSize != 1 { + t.Errorf("expected 1 task in dead letter queue after exceeding retries, got %d", dlqSize) + } +} + +func TestRescheduleTask_NoOutcomeUsesDefaultInterval(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 45 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 0, // no interval set + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // No outcome recorded - hasOutcome will be false + m.rescheduleTask(task) + + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected task to be rescheduled") + } + + // Should use config PVE polling interval + if entry.task.Interval != cfg.PVEPollingInterval { + t.Errorf("expected interval %v, got %v", cfg.PVEPollingInterval, entry.task.Interval) + } +} + +func TestRescheduleTask_PBSInstance(t *testing.T) { + cfg := &config.Config{ + PBSPollingInterval: 60 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + } + + task := ScheduledTask{ + InstanceName: "pbs-1", + InstanceType: InstanceTypePBS, + Interval: 0, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + m.rescheduleTask(task) + + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected PBS task to be rescheduled") + } + + if entry.task.Interval != cfg.PBSPollingInterval { + t.Errorf("expected PBS interval %v, got %v", cfg.PBSPollingInterval, entry.task.Interval) + } +} + +func TestRescheduleTask_PMGInstance(t *testing.T) { + cfg := &config.Config{ + PMGPollingInterval: 90 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + } + + task := ScheduledTask{ + InstanceName: "pmg-1", + InstanceType: InstanceTypePMG, + Interval: 0, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + m.rescheduleTask(task) + + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected PMG task to be rescheduled") + } + + if entry.task.Interval != cfg.PMGPollingInterval { + t.Errorf("expected PMG interval %v, got %v", cfg.PMGPollingInterval, entry.task.Interval) + } +} + +func TestRescheduleTask_AdaptivePollingMaxIntervalLimit(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + AdaptivePollingEnabled: true, + AdaptivePollingMaxInterval: 10 * time.Second, // <= 15s enables capping + AdaptivePollingBaseInterval: 5 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + maxRetryAttempts: 5, + backoffCfg: backoffConfig{ + Initial: 10 * time.Second, // would normally backoff to 10s+ + Multiplier: 2, + Jitter: 0, + Max: 5 * time.Minute, + }, + } + + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: 30 * time.Second, + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + // Simulate transient failure to trigger backoff + m.failureCounts[key] = 1 + m.lastOutcome[key] = taskOutcome{ + success: false, + transient: true, + err: errors.New("timeout"), + } + + m.rescheduleTask(task) + + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected task to be rescheduled") + } + + // With AdaptivePollingMaxInterval <= 15s, backoff delay should be capped at 4s + maxDelay := 4 * time.Second + if entry.task.Interval > maxDelay { + t.Errorf("expected backoff interval to be capped at %v, got %v", maxDelay, entry.task.Interval) + } +} + +func TestRescheduleTask_UsesExistingIntervalWhenSet(t *testing.T) { + cfg := &config.Config{ + PVEPollingInterval: 30 * time.Second, + AdaptivePollingBaseInterval: 10 * time.Second, + } + + m := &Monitor{ + config: cfg, + taskQueue: NewTaskQueue(), + lastOutcome: make(map[string]taskOutcome), + failureCounts: make(map[string]int), + } + + customInterval := 45 * time.Second + task := ScheduledTask{ + InstanceName: "pve-1", + InstanceType: InstanceTypePVE, + Interval: customInterval, // custom interval already set + NextRun: time.Now(), + } + + key := schedulerKey(task.InstanceType, task.InstanceName) + + m.rescheduleTask(task) + + m.taskQueue.mu.Lock() + entry, ok := m.taskQueue.entries[key] + m.taskQueue.mu.Unlock() + + if !ok { + t.Fatal("expected task to be rescheduled") + } + + // Should use the existing interval when it's already set + if entry.task.Interval != customInterval { + t.Errorf("expected existing interval %v to be preserved, got %v", customInterval, entry.task.Interval) + } +}