Guard PBS backups from failed polls

Related to #613

When all PBS datastore queries fail (e.g., due to network issues or PBS
downtime), the system was clearing all backups and showing an empty list.
This adds the same preservation logic that exists for PVE storage backups.

Changes:
- Add shouldPreservePBSBackups() helper function
- Track datastore query success/failure counts in pollPBSBackups()
- Preserve existing backups when all datastore queries fail
- Add comprehensive unit tests for PBS backup preservation logic

This ensures users can still see their backup history even during
temporary connectivity issues with PBS, matching the behavior already
implemented for PVE storage backups.
This commit is contained in:
rcourtman 2025-11-05 19:26:20 +00:00
parent 404d461d35
commit fcba710183
2 changed files with 71 additions and 0 deletions

View file

@ -66,3 +66,50 @@ func TestShouldPreserveBackups(t *testing.T) {
})
}
}
func TestShouldPreservePBSBackups(t *testing.T) {
t.Parallel()
tests := []struct {
name string
datastoreCount int
datastoreFetches int
want bool
}{
{
name: "all datastores failed",
datastoreCount: 3,
datastoreFetches: 0,
want: true,
},
{
name: "no datastores skips preservation",
datastoreCount: 0,
datastoreFetches: 0,
want: false,
},
{
name: "some datastores succeeded",
datastoreCount: 3,
datastoreFetches: 2,
want: false,
},
{
name: "all datastores succeeded",
datastoreCount: 3,
datastoreFetches: 3,
want: false,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := shouldPreservePBSBackups(tt.datastoreCount, tt.datastoreFetches)
if got != tt.want {
t.Fatalf("shouldPreservePBSBackups() = %v, want %v", got, tt.want)
}
})
}
}

View file

@ -7928,6 +7928,14 @@ func shouldPreserveBackups(nodeCount int, hadSuccessfulNode bool, storagesWithBa
return false
}
func shouldPreservePBSBackups(datastoreCount, datastoreFetches int) bool {
// If there are datastores but all fetches failed, preserve existing backups
if datastoreCount > 0 && datastoreFetches == 0 {
return true
}
return false
}
func buildGuestLookups(snapshot models.StateSnapshot) (map[string]alerts.GuestLookup, map[string]alerts.GuestLookup) {
byKey := make(map[string]alerts.GuestLookup)
byVMID := make(map[string]alerts.GuestLookup)
@ -8507,6 +8515,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
log.Debug().Str("instance", instanceName).Msg("Polling PBS backups")
var allBackups []models.PBSBackup
datastoreCount := len(datastores) // Number of datastores to query
datastoreFetches := 0 // Number of successful datastore fetches
datastoreErrors := 0 // Number of failed datastore fetches
// Process each datastore
for _, ds := range datastores {
@ -8530,9 +8541,12 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
Str("instance", instanceName).
Str("datastore", ds.Name).
Msg("Failed to fetch PBS backups")
datastoreErrors++
continue
}
datastoreFetches++
// Convert PBS backups to model backups
for namespace, snapshots := range backupsMap {
for _, snapshot := range snapshots {
@ -8604,6 +8618,16 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
Int("count", len(allBackups)).
Msg("PBS backups fetched")
// Decide whether to keep existing backups when all queries failed
if shouldPreservePBSBackups(datastoreCount, datastoreFetches) {
log.Warn().
Str("instance", instanceName).
Int("datastores", datastoreCount).
Int("errors", datastoreErrors).
Msg("All PBS datastore queries failed; keeping previous backup list")
return
}
// Update state
m.state.UpdatePBSBackups(instanceName, allBackups)