Fix P1/P2 infrastructure issues: panic recovery and optimizations
This commit addresses 4 P1 important issues and 1 P2 optimization in infrastructure components: **P1-1: Missing Panic Recovery in Discovery Service** (service.go:172-195, 499-542) - **Problem**: No panic recovery in Start(), ForceRefresh(), SetSubnet() goroutines - **Impact**: Silent service death if scan panics, broken discovery with no monitoring - **Fix**: - Wrapped initial scan goroutine with defer/recover (lines 172-182) - Wrapped scanLoop goroutine with defer/recover (lines 185-195) - Wrapped ForceRefresh scan with defer/recover (lines 499-509) - Wrapped SetSubnet scan with defer/recover (lines 532-542) - All log panics with stack traces for debugging **P1-2: Missing Panic Recovery in Config Watcher Callback** (watcher.go:546-556) - **Problem**: User-provided onMockReload callback could panic and crash watcher - **Impact**: Panicking callback kills watcher goroutine, no config updates - **Fix**: Wrapped callback invocation with defer/recover and stack trace logging **P1-3: Session Store Stop() Using Send Instead of Close** (session_store.go:16-84) - **Problem**: Stop() used channel send which blocks if nobody reads - **Impact**: Stop() hangs if backgroundWorker already exited - **Fix**: - Added sync.Once field stopOnce (line 22) - Changed Stop() to use close() within stopOnce.Do() (lines 80-84) - Prevents double-close panic and ensures all readers are signaled **P2-1: Backup Cleanup Inefficient O(n²) Sort** (persistence.go:1424-1427) - **Problem**: Bubble sort used to sort backups by modification time - **Impact**: Inefficient for large backup counts (>100 files) - **Fix**: - Replaced bubble sort with sort.Slice() using O(n log n) algorithm - Added "sort" import (line 9) - Maintains same oldest-first ordering for deletion logic All fixes add defensive programming without changing external behavior. Panic recovery ensures services continue operating even with bugs, while optimization reduces cleanup time for backup-heavy environments.
This commit is contained in:
parent
ba6d934204
commit
6ca4d9b750
4 changed files with 68 additions and 17 deletions
|
|
@ -19,6 +19,7 @@ type SessionStore struct {
|
||||||
dataPath string
|
dataPath string
|
||||||
saveTicker *time.Ticker
|
saveTicker *time.Ticker
|
||||||
stopChan chan bool
|
stopChan chan bool
|
||||||
|
stopOnce sync.Once // Ensures Stop() can only close channel once
|
||||||
}
|
}
|
||||||
|
|
||||||
func sessionHash(token string) string {
|
func sessionHash(token string) string {
|
||||||
|
|
@ -76,9 +77,11 @@ func (s *SessionStore) backgroundWorker() {
|
||||||
|
|
||||||
// Stop gracefully stops the session store
|
// Stop gracefully stops the session store
|
||||||
func (s *SessionStore) Stop() {
|
func (s *SessionStore) Stop() {
|
||||||
s.saveTicker.Stop()
|
s.stopOnce.Do(func() {
|
||||||
s.stopChan <- true
|
s.saveTicker.Stop()
|
||||||
s.save()
|
close(s.stopChan) // Use close instead of send to signal all readers
|
||||||
|
s.save()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateSession creates a new session
|
// CreateSession creates a new session
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
@ -1421,14 +1422,10 @@ func (c *ConfigPersistence) cleanupOldBackups(pattern string) {
|
||||||
files = append(files, fileInfo{path: match, modTime: info.ModTime()})
|
files = append(files, fileInfo{path: match, modTime: info.ModTime()})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort oldest first
|
// Sort oldest first using efficient sort
|
||||||
for i := 0; i < len(files)-1; i++ {
|
sort.Slice(files, func(i, j int) bool {
|
||||||
for j := i + 1; j < len(files); j++ {
|
return files[i].modTime.Before(files[j].modTime)
|
||||||
if files[i].modTime.After(files[j].modTime) {
|
})
|
||||||
files[i], files[j] = files[j], files[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete oldest backups (keep last 10)
|
// Delete oldest backups (keep last 10)
|
||||||
toDelete := len(files) - maxBackups
|
toDelete := len(files) - maxBackups
|
||||||
|
|
|
||||||
|
|
@ -543,6 +543,16 @@ func (cw *ConfigWatcher) reloadMockConfig() {
|
||||||
// Trigger callback to restart backend if set
|
// Trigger callback to restart backend if set
|
||||||
if callback != nil {
|
if callback != nil {
|
||||||
log.Info().Msg("Triggering backend restart due to mock.env change")
|
log.Info().Msg("Triggering backend restart due to mock.env change")
|
||||||
go callback()
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Stack().
|
||||||
|
Msg("Recovered from panic in mock.env callback")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
callback()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -168,10 +168,31 @@ func (s *Service) Start(ctx context.Context) {
|
||||||
Msg("Starting background discovery service")
|
Msg("Starting background discovery service")
|
||||||
|
|
||||||
// Do initial scan immediately
|
// Do initial scan immediately
|
||||||
go s.performScan()
|
// Initial scan with panic recovery
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Stack().
|
||||||
|
Msg("Recovered from panic in initial discovery scan")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
s.performScan()
|
||||||
|
}()
|
||||||
|
|
||||||
// Start background scanning loop
|
// Start background scanning loop with panic recovery
|
||||||
go s.scanLoop()
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Stack().
|
||||||
|
Msg("Recovered from panic in discovery scan loop")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
s.scanLoop()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the background discovery service
|
// Stop stops the background discovery service
|
||||||
|
|
@ -475,7 +496,17 @@ func (s *Service) ForceRefresh() {
|
||||||
}
|
}
|
||||||
s.mu.RUnlock()
|
s.mu.RUnlock()
|
||||||
|
|
||||||
go s.performScan()
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Stack().
|
||||||
|
Msg("Recovered from panic in ForceRefresh scan")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
s.performScan()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetInterval updates the scan interval
|
// SetInterval updates the scan interval
|
||||||
|
|
@ -498,7 +529,17 @@ func (s *Service) SetSubnet(subnet string) {
|
||||||
|
|
||||||
// Trigger immediate rescan with new subnet if not already scanning
|
// Trigger immediate rescan with new subnet if not already scanning
|
||||||
if !alreadyScanning {
|
if !alreadyScanning {
|
||||||
go s.performScan()
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Stack().
|
||||||
|
Msg("Recovered from panic in SetSubnet scan")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
s.performScan()
|
||||||
|
}()
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Msg("Scan already in progress, new subnet will be used in next scan")
|
log.Debug().Msg("Scan already in progress, new subnet will be used in next scan")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue