Fix CSRF token validation and improve token management

- Add Access-Control-Expose-Headers to allow frontend to read X-CSRF-Token response header
- Implement proactive CSRF token issuance on GET requests when session exists but CSRF cookie is missing
- Ensures frontend always has valid CSRF token before making POST requests
- Fixes 403 Forbidden errors when toggling system settings

This resolves CSRF validation failures that occurred when CSRF tokens expired or were missing while valid sessions existed.
This commit is contained in:
rcourtman 2025-11-05 09:23:44 +00:00
parent 10862db4e4
commit d52ac6d8b5
14 changed files with 536 additions and 123 deletions

View file

@ -117,6 +117,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout
"allowedOrigins": "", // CORS allowed origins (empty = same-origin only)
"allowEmbedding": false, // Allow iframe embedding
"allowedEmbedOrigins": "", // Comma-separated origins allowed to embed Pulse
"temperatureMonitoringEnabled": true,// Global temperature polling toggle (Settings → Proxmox → Edit node → Advanced monitoring)
"backendPort": 3000, // Internal API listen port (not normally changed)
"frontendPort": 7655, // Public port exposed by the service
"logLevel": "info", // Log level: debug, info, warn, error
@ -388,6 +389,7 @@ These env vars override system.json values. When set, the UI will show a warning
- `ADAPTIVE_POLLING_MAX_INTERVAL` - Override the maximum cadence (Go duration or seconds). Values ≤`15s` engage the low-latency backoff profile.
- `ENABLE_BACKUP_POLLING` - Set to `false` to disable polling of Proxmox backup/snapshot APIs (default: true)
- `BACKUP_POLLING_INTERVAL` - Override the backup polling cadence. Accepts Go duration syntax (e.g. `30m`, `6h`) or seconds. Use `0` for Pulse's default (~90s) cadence.
- `ENABLE_TEMPERATURE_MONITORING` - Force-enable or disable SSH temperature polling for all nodes (`true`/`false`)
- `PULSE_PUBLIC_URL` - Full URL to access Pulse (e.g., `http://192.168.1.100:7655`)
- **Auto-detected** if not set (except inside Docker where detection is disabled)
- Used in webhook notifications for "View in Pulse" links

View file

@ -12,6 +12,19 @@ Pulse can display real-time CPU and NVMe temperatures directly in your dashboard
- Yellow: 60-80°C (warm)
- Red: > 80°C (hot)
## Disable Temperature Monitoring
Don't need the sensor data? Open **Settings → Proxmox**, edit any node, and scroll to the **Advanced monitoring** section. The temperature toggle there controls collection for all nodes:
- When disabled, Pulse skips every SSH/proxy request for temperature data.
- CPU and NVMe readings disappear from dashboards and node tables.
- You can re-enable it later without re-running the setup scripts.
For scripted environments, set either:
- `temperatureMonitoringEnabled: false` in `/etc/pulse/system.json`, or
- `ENABLE_TEMPERATURE_MONITORING=false` in the environment (locks the UI toggle until removed).
## How It Works
### Secure Architecture (v4.24.0+)

View file

@ -15,6 +15,7 @@ import {
formCheckbox,
} from '@/components/shared/Form';
import { logger } from '@/utils/logger';
import { TogglePrimitive } from '@/components/shared/Toggle';
interface NodeModalProps {
isOpen: boolean;
@ -26,6 +27,10 @@ interface NodeModalProps {
showBackToDiscovery?: boolean;
onBackToDiscovery?: () => void;
securityStatus?: Partial<SecurityStatus>;
temperatureMonitoringEnabled?: boolean;
temperatureMonitoringLocked?: boolean;
savingTemperatureSetting?: boolean;
onToggleTemperatureMonitoring?: (enabled: boolean) => Promise<void> | void;
}
const deriveNameFromHost = (host: string): string => {
@ -1705,28 +1710,68 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
{/* Physical Disk Monitoring - PVE only */}
<Show when={props.nodeType === 'pve'}>
<div>
<div class="space-y-4">
<SectionHeader
title="Advanced monitoring"
size="sm"
class="mb-3"
titleClass="text-gray-900 dark:text-gray-100"
/>
<label class="flex items-start gap-2 text-sm text-gray-700 dark:text-gray-300">
<input
type="checkbox"
checked={formData().monitorPhysicalDisks}
onChange={(e) => updateField('monitorPhysicalDisks', e.currentTarget.checked)}
class={formCheckbox + ' mt-0.5'}
/>
<div>
<div>Monitor physical disk health (SMART)</div>
<p class="text-xs text-gray-500 dark:text-gray-400 mt-1">
Polls disk SMART data every 5 minutes. Note: This will cause HDDs to spin up from standby.
If you have HDDs that should stay idle, leave this disabled.
</p>
<div class="rounded-lg border border-gray-200 bg-white p-3 text-sm text-gray-700 shadow-sm dark:border-gray-700 dark:bg-gray-800 dark:text-gray-200">
<div class="flex items-start justify-between gap-3">
<div>
<p class="font-medium text-gray-900 dark:text-gray-100">Monitor physical disk health (SMART)</p>
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
Polls disk SMART data every 5 minutes. This will spin up idle HDDs; leave disabled if you rely on drive standby.
</p>
</div>
<TogglePrimitive
checked={formData().monitorPhysicalDisks}
onChange={(event) => updateField('monitorPhysicalDisks', event.currentTarget.checked)}
ariaLabel={
formData().monitorPhysicalDisks
? 'Disable physical disk monitoring'
: 'Enable physical disk monitoring'
}
/>
</div>
</label>
</div>
<Show when={typeof props.temperatureMonitoringEnabled === 'boolean'}>
{() => {
const enabled = props.temperatureMonitoringEnabled ?? true;
return (
<div class="rounded-lg border border-gray-200 bg-white p-3 text-sm text-gray-700 shadow-sm dark:border-gray-700 dark:bg-gray-800 dark:text-gray-200">
<div class="flex items-start justify-between gap-3">
<div>
<p class="font-medium text-gray-900 dark:text-gray-100">Temperature monitoring</p>
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
Uses the Pulse sensors key or proxy to read CPU/NVMe temperatures for every node. Disable if you dont need temperature data or havent deployed the proxy yet.
</p>
</div>
<TogglePrimitive
checked={enabled}
onChange={(event) => {
props.onToggleTemperatureMonitoring?.(event.currentTarget.checked);
}}
disabled={props.temperatureMonitoringLocked || props.savingTemperatureSetting}
ariaLabel={enabled ? 'Disable temperature monitoring' : 'Enable temperature monitoring'}
/>
</div>
<Show when={!enabled}>
<p class="mt-3 rounded border border-blue-200 bg-blue-50 p-2 text-xs text-blue-700 dark:border-blue-700 dark:bg-blue-900/20 dark:text-blue-200">
Pulse will skip all SSH temperature polling until you re-enable this toggle. Existing dashboard readings will stop refreshing.
</p>
</Show>
<Show when={props.temperatureMonitoringLocked}>
<p class="mt-3 rounded border border-amber-200 bg-amber-50 p-2 text-xs text-amber-800 dark:border-amber-700 dark:bg-amber-900/20 dark:text-amber-200">
Locked by environment variables. Remove the override (ENABLE_TEMPERATURE_MONITORING) and restart Pulse to manage it in the UI.
</p>
</Show>
</div>
);
}}
</Show>
</div>
</Show>
<Show when={props.nodeType === 'pmg'}>

View file

@ -545,6 +545,12 @@ const Settings: Component<SettingsProps> = (props) => {
const [discoverySubnetError, setDiscoverySubnetError] = createSignal<string | undefined>();
const [savingDiscoverySettings, setSavingDiscoverySettings] = createSignal(false);
const [envOverrides, setEnvOverrides] = createSignal<Record<string, boolean>>({});
const [temperatureMonitoringEnabled, setTemperatureMonitoringEnabled] = createSignal(true);
const [savingTemperatureSetting, setSavingTemperatureSetting] = createSignal(false);
const temperatureMonitoringLocked = () =>
Boolean(
envOverrides().temperatureMonitoringEnabled || envOverrides()['ENABLE_TEMPERATURE_MONITORING'],
);
let discoverySubnetInputRef: HTMLInputElement | undefined;
const parseSubnetList = (value: string) => {
@ -1378,6 +1384,31 @@ const Settings: Component<SettingsProps> = (props) => {
}
};
const handleTemperatureMonitoringChange = async (enabled: boolean): Promise<void> => {
if (temperatureMonitoringLocked() || savingTemperatureSetting()) {
return;
}
const previous = temperatureMonitoringEnabled();
setTemperatureMonitoringEnabled(enabled);
setSavingTemperatureSetting(true);
try {
await SettingsAPI.updateSystemSettings({ temperatureMonitoringEnabled: enabled });
if (enabled) {
notificationStore.success('Temperature monitoring enabled', 2000);
} else {
notificationStore.info('Temperature monitoring disabled', 2000);
}
} catch (error) {
logger.error('Failed to update temperature monitoring setting', error);
notificationStore.error('Failed to update temperature monitoring setting');
setTemperatureMonitoringEnabled(previous);
} finally {
setSavingTemperatureSetting(false);
}
};
const handleDiscoveryModeChange = async (mode: 'auto' | 'custom') => {
if (envOverrides().discoverySubnet || savingDiscoverySettings()) {
return;
@ -1567,6 +1598,11 @@ const Settings: Component<SettingsProps> = (props) => {
// Load embedding settings
setAllowEmbedding(systemSettings.allowEmbedding ?? false);
setAllowedEmbedOrigins(systemSettings.allowedEmbedOrigins || '');
setTemperatureMonitoringEnabled(
typeof systemSettings.temperatureMonitoringEnabled === 'boolean'
? systemSettings.temperatureMonitoringEnabled
: true,
);
// Backup polling controls
if (typeof systemSettings.backupPollingEnabled === 'boolean') {
setBackupPollingEnabled(systemSettings.backupPollingEnabled);
@ -6536,6 +6572,10 @@ const Settings: Component<SettingsProps> = (props) => {
nodeType="pve"
editingNode={editingNode()?.type === 'pve' ? (editingNode() ?? undefined) : undefined}
securityStatus={securityStatus() ?? undefined}
temperatureMonitoringEnabled={temperatureMonitoringEnabled()}
temperatureMonitoringLocked={temperatureMonitoringLocked()}
savingTemperatureSetting={savingTemperatureSetting()}
onToggleTemperatureMonitoring={handleTemperatureMonitoringChange}
onSave={async (nodeData) => {
try {
if (editingNode() && editingNode()!.id) {

View file

@ -36,6 +36,7 @@ export interface SystemConfig {
autoUpdateTime?: string; // Time for updates (HH:MM format)
backupPollingInterval?: number; // Backup polling interval in seconds (0 = default cadence)
backupPollingEnabled?: boolean; // Enable backup polling of PVE/PBS data
temperatureMonitoringEnabled?: boolean; // Collect CPU/NVMe temperatures via SSH
allowedOrigins?: string; // CORS allowed origins
backendPort?: number; // Backend API port (default: 7655)
frontendPort?: number; // Frontend UI port (default: 7655)
@ -154,6 +155,7 @@ export const DEFAULT_CONFIG: {
autoUpdateTime: '03:00',
backupPollingEnabled: true,
backupPollingInterval: 0,
temperatureMonitoringEnabled: true,
allowedOrigins: '',
backendPort: 7655,
frontendPort: 7655,

View file

@ -60,16 +60,18 @@ class ApiClient {
}
}
private loadCSRFToken() {
private loadCSRFToken(): string | null {
// Read CSRF token from cookie
const cookies = document.cookie.split(';');
for (const cookie of cookies) {
const [name, value] = cookie.trim().split('=');
if (name === 'pulse_csrf') {
this.csrfToken = decodeURIComponent(value);
break;
}
const [name, ...rest] = cookie.trim().split('=');
if (name !== 'pulse_csrf') continue;
const value = rest.join('=');
this.csrfToken = decodeURIComponent(value || '');
return this.csrfToken;
}
this.csrfToken = null;
return null;
}
private loadStoredAuth() {
@ -201,8 +203,11 @@ class ApiClient {
// Add CSRF token for state-changing requests
const method = (fetchOptions.method || 'GET').toUpperCase();
if (this.csrfToken && method !== 'GET' && method !== 'HEAD' && method !== 'OPTIONS') {
finalHeaders['X-CSRF-Token'] = this.csrfToken;
if (method !== 'GET' && method !== 'HEAD' && method !== 'OPTIONS') {
const token = this.loadCSRFToken();
if (token) {
finalHeaders['X-CSRF-Token'] = token;
}
}
// Always include credentials for cookies (WebSocket session support)
@ -223,19 +228,24 @@ class ApiClient {
// Handle CSRF token failures
if (response.status === 403) {
const text = await response.clone().text();
if (text.includes('CSRF')) {
// Try to reload CSRF token from cookie and retry
this.loadCSRFToken();
if (this.csrfToken) {
finalHeaders['X-CSRF-Token'] = this.csrfToken;
const retryResponse = await fetch(url, {
...fetchOptions,
headers: finalHeaders,
credentials: 'include',
});
return retryResponse;
}
const csrfHeader = response.headers.get('X-CSRF-Token');
let refreshedToken: string | null = null;
if (csrfHeader) {
refreshedToken = csrfHeader;
} else {
refreshedToken = this.loadCSRFToken();
}
if (refreshedToken) {
this.csrfToken = refreshedToken;
logger.debug(`[apiClient] Retrying ${method} ${url} with refreshed CSRF token`);
finalHeaders['X-CSRF-Token'] = refreshedToken;
const retryResponse = await fetch(url, {
...fetchOptions,
headers: finalHeaders,
credentials: 'include',
});
return retryResponse;
}
}

View file

@ -1181,6 +1181,7 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", r.config.AllowedOrigins)
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-API-Token, X-CSRF-Token")
w.Header().Set("Access-Control-Expose-Headers", "X-CSRF-Token, X-Authenticated-User, X-Auth-Method")
}
// Handle preflight requests
@ -1345,6 +1346,29 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
// Issue CSRF token for GET requests if session exists but CSRF cookie is missing
// This ensures the frontend has a token before making POST requests
if req.Method == "GET" && strings.HasPrefix(req.URL.Path, "/api/") {
sessionCookie, err := req.Cookie("pulse_session")
if err == nil && sessionCookie.Value != "" {
// Check if CSRF cookie exists
_, csrfErr := req.Cookie("pulse_csrf")
if csrfErr != nil {
// Session exists but no CSRF cookie - issue one
csrfToken := generateCSRFToken(sessionCookie.Value)
isSecure, sameSitePolicy := getCookieSettings(req)
http.SetCookie(w, &http.Cookie{
Name: "pulse_csrf",
Value: csrfToken,
Path: "/",
Secure: isSecure,
SameSite: sameSitePolicy,
MaxAge: 86400,
})
}
}
}
// Rate limiting is now handled by UniversalRateLimitMiddleware
// No need for duplicate rate limiting logic here

View file

@ -61,6 +61,9 @@ func CheckCSRF(w http.ResponseWriter, r *http.Request) bool {
Str("session", cookie.Value[:8]+"...").
Msg("Missing CSRF token")
clearCSRFCookie(w)
if newToken := issueNewCSRFCookie(w, r, cookie.Value); newToken != "" {
w.Header().Set("X-CSRF-Token", newToken)
}
return false
}
@ -72,6 +75,9 @@ func CheckCSRF(w http.ResponseWriter, r *http.Request) bool {
Str("provided_token", csrfToken[:8]+"...").
Msg("Invalid CSRF token")
clearCSRFCookie(w)
if newToken := issueNewCSRFCookie(w, r, cookie.Value); newToken != "" {
w.Header().Set("X-CSRF-Token", newToken)
}
return false
}
@ -91,6 +97,28 @@ func clearCSRFCookie(w http.ResponseWriter) {
})
}
func issueNewCSRFCookie(w http.ResponseWriter, r *http.Request, sessionID string) string {
if w == nil || r == nil {
return ""
}
if strings.TrimSpace(sessionID) == "" {
return ""
}
newToken := generateCSRFToken(sessionID)
secure, sameSite := getCookieSettings(r)
http.SetCookie(w, &http.Cookie{
Name: "pulse_csrf",
Value: newToken,
Path: "/",
Secure: secure,
SameSite: sameSite,
MaxAge: 86400,
})
return newToken
}
// Rate Limiting - using existing RateLimiter from ratelimit.go
var (
// Auth endpoints: 10 attempts per minute

View file

@ -31,6 +31,8 @@ type SystemSettingsHandler struct {
GetDiscoveryService() *discovery.Service
StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string)
StopDiscoveryService()
EnableTemperatureMonitoring()
DisableTemperatureMonitoring()
}
}
@ -39,6 +41,8 @@ func NewSystemSettingsHandler(cfg *config.Config, persistence *config.ConfigPers
GetDiscoveryService() *discovery.Service
StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string)
StopDiscoveryService()
EnableTemperatureMonitoring()
DisableTemperatureMonitoring()
}, reloadSystemSettingsFunc func()) *SystemSettingsHandler {
return &SystemSettingsHandler{
config: cfg,
@ -54,6 +58,8 @@ func (h *SystemSettingsHandler) SetMonitor(m interface {
GetDiscoveryService() *discovery.Service
StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string)
StopDiscoveryService()
EnableTemperatureMonitoring()
DisableTemperatureMonitoring()
}) {
h.monitor = m
}
@ -181,6 +187,12 @@ func validateSystemSettings(settings *config.SystemSettings, rawRequest map[stri
}
}
if val, ok := rawRequest["temperatureMonitoringEnabled"]; ok {
if _, ok := val.(bool); !ok {
return fmt.Errorf("temperatureMonitoringEnabled must be a boolean")
}
}
// Validate auto-update check interval (min 1 hour, max 7 days)
if val, ok := rawRequest["autoUpdateCheckInterval"]; ok {
if interval, ok := val.(float64); ok {
@ -367,6 +379,7 @@ func (h *SystemSettingsHandler) HandleGetSystemSettings(w http.ResponseWriter, r
enabled := h.config.EnableBackupPolling
settings.BackupPollingEnabled = &enabled
settings.DiscoveryConfig = config.CloneDiscoveryConfig(h.config.Discovery)
settings.TemperatureMonitoringEnabled = h.config.TemperatureMonitoringEnabled
}
// Include env override information
@ -463,6 +476,8 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter
// Start with existing settings
settings := *existingSettings
discoveryConfigUpdated := false
prevTempEnabled := h.config.TemperatureMonitoringEnabled
tempToggleRequested := false
// Only update fields that were provided in the request
// Note: PVE polling is hardcoded to 10s, legacy polling fields are ignored
@ -548,6 +563,10 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter
if _, ok := rawRequest["backupPollingEnabled"]; ok {
settings.BackupPollingEnabled = updates.BackupPollingEnabled
}
if _, ok := rawRequest["temperatureMonitoringEnabled"]; ok {
settings.TemperatureMonitoringEnabled = updates.TemperatureMonitoringEnabled
tempToggleRequested = true
}
// Update the config
// Note: PVE polling is hardcoded to 10s
@ -597,6 +616,10 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter
}
h.config.Discovery = config.CloneDiscoveryConfig(settings.DiscoveryConfig)
if tempToggleRequested {
h.config.TemperatureMonitoringEnabled = settings.TemperatureMonitoringEnabled
}
// Start or stop discovery service based on setting change
if h.monitor != nil {
if settings.DiscoveryEnabled && !prevDiscoveryEnabled {
@ -625,6 +648,14 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter
}
}
if tempToggleRequested && h.monitor != nil {
if settings.TemperatureMonitoringEnabled && !prevTempEnabled {
h.monitor.EnableTemperatureMonitoring()
} else if !settings.TemperatureMonitoringEnabled && prevTempEnabled {
h.monitor.DisableTemperatureMonitoring()
}
}
// Save to persistence
if err := h.persistence.SaveSystemSettings(settings); err != nil {
log.Error().Err(err).Msg("Failed to save system settings")

View file

@ -91,6 +91,7 @@ type Config struct {
BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"`
BackupPollingInterval time.Duration `envconfig:"BACKUP_POLLING_INTERVAL"`
EnableBackupPolling bool `envconfig:"ENABLE_BACKUP_POLLING" default:"true"`
TemperatureMonitoringEnabled bool `json:"temperatureMonitoringEnabled"`
WebhookBatchDelay time.Duration `envconfig:"WEBHOOK_BATCH_DELAY" default:"10s"`
AdaptivePollingEnabled bool `envconfig:"ADAPTIVE_POLLING_ENABLED" default:"false"`
AdaptivePollingBaseInterval time.Duration `envconfig:"ADAPTIVE_POLLING_BASE_INTERVAL" default:"10s"`
@ -528,6 +529,7 @@ func Load() (*Config, error) {
PMGPollingInterval: 60 * time.Second, // Default PMG polling (aggregated stats)
DiscoveryEnabled: false,
DiscoverySubnet: "auto",
TemperatureMonitoringEnabled: true,
EnvOverrides: make(map[string]bool),
OIDC: NewOIDCConfig(),
}
@ -606,10 +608,11 @@ func Load() (*Config, error) {
// Always load DiscoveryEnabled even if false
cfg.DiscoveryEnabled = systemSettings.DiscoveryEnabled
if systemSettings.DiscoverySubnet != "" {
cfg.DiscoverySubnet = systemSettings.DiscoverySubnet
}
cfg.Discovery = NormalizeDiscoveryConfig(CloneDiscoveryConfig(systemSettings.DiscoveryConfig))
// APIToken no longer loaded from system.json - only from .env
cfg.DiscoverySubnet = systemSettings.DiscoverySubnet
}
cfg.Discovery = NormalizeDiscoveryConfig(CloneDiscoveryConfig(systemSettings.DiscoveryConfig))
cfg.TemperatureMonitoringEnabled = systemSettings.TemperatureMonitoringEnabled
// APIToken no longer loaded from system.json - only from .env
log.Info().
Str("updateChannel", cfg.UpdateChannel).
Str("logLevel", cfg.LogLevel).
@ -688,6 +691,20 @@ func Load() (*Config, error) {
}
}
if enabledStr := utils.GetenvTrim("ENABLE_TEMPERATURE_MONITORING"); enabledStr != "" {
if enabled, err := strconv.ParseBool(enabledStr); err == nil {
cfg.TemperatureMonitoringEnabled = enabled
cfg.EnvOverrides["temperatureMonitoringEnabled"] = true
log.Info().
Bool("enabled", enabled).
Msg("Overriding temperature monitoring setting from environment")
} else {
log.Warn().
Str("value", enabledStr).
Msg("Invalid ENABLE_TEMPERATURE_MONITORING value, ignoring")
}
}
if enabledStr := utils.GetenvTrim("ENABLE_BACKUP_POLLING"); enabledStr != "" {
switch strings.ToLower(enabledStr) {
case "0", "false", "no", "off":

View file

@ -820,6 +820,7 @@ type SystemSettings struct {
Theme string `json:"theme,omitempty"` // User theme preference: "light", "dark", or empty for system default
AllowEmbedding bool `json:"allowEmbedding"` // Allow iframe embedding
AllowedEmbedOrigins string `json:"allowedEmbedOrigins,omitempty"` // Comma-separated list of allowed origins for embedding
TemperatureMonitoringEnabled bool `json:"temperatureMonitoringEnabled"`
// APIToken removed - now handled via .env file only
}
@ -834,6 +835,7 @@ func DefaultSystemSettings() *SystemSettings {
DiscoverySubnet: "auto",
DiscoveryConfig: defaultDiscovery,
AllowEmbedding: false,
TemperatureMonitoringEnabled: true,
}
}

View file

@ -439,7 +439,7 @@ type Monitor struct {
backoffCfg backoffConfig
rng *rand.Rand
maxRetryAttempts int
tempCollector *TemperatureCollector // SSH-based temperature collector
tempService TemperatureService
mu sync.RWMutex
startTime time.Time
rateTracker *RateTracker
@ -486,6 +486,55 @@ type Monitor struct {
dlqInsightMap map[string]*dlqInsight
}
func (m *Monitor) temperatureService() TemperatureService {
if m == nil {
return nil
}
m.mu.RLock()
service := m.tempService
m.mu.RUnlock()
return service
}
// EnableTemperatureMonitoring reenables temperature collection and ensures the provider is available.
func (m *Monitor) EnableTemperatureMonitoring() {
if m == nil {
return
}
m.mu.Lock()
service := m.tempService
defer m.mu.Unlock()
if m.config != nil {
m.config.TemperatureMonitoringEnabled = true
}
if service != nil {
service.Enable()
}
}
// DisableTemperatureMonitoring stops temperature collection attempts.
func (m *Monitor) DisableTemperatureMonitoring() {
if m == nil {
return
}
m.mu.Lock()
service := m.tempService
defer m.mu.Unlock()
if m.config != nil {
m.config.TemperatureMonitoringEnabled = false
}
if service != nil {
service.Disable()
}
}
type rrdMemCacheEntry struct {
available uint64
used uint64
@ -3012,10 +3061,12 @@ func New(cfg *config.Config) (*Monitor, error) {
homeDir = "/home/pulse"
}
sshKeyPath := filepath.Join(homeDir, ".ssh/id_ed25519_sensors")
tempCollector := NewTemperatureCollector("root", sshKeyPath)
tempService := newTemperatureService(cfg.TemperatureMonitoringEnabled, "root", sshKeyPath)
// Security warning if running in container with SSH temperature monitoring
checkContainerizedTempMonitoring()
if cfg.TemperatureMonitoringEnabled {
checkContainerizedTempMonitoring()
}
stalenessTracker := NewStalenessTracker(getPollMetrics())
stalenessTracker.SetBounds(cfg.AdaptivePollingBaseInterval, cfg.AdaptivePollingMaxInterval)
@ -3080,7 +3131,7 @@ func New(cfg *config.Config) (*Monitor, error) {
backoffCfg: backoff,
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
maxRetryAttempts: 5,
tempCollector: tempCollector,
tempService: tempService,
startTime: time.Now(),
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
@ -5185,97 +5236,103 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
// Collect temperature data via SSH (non-blocking, best effort)
// Only attempt for online nodes
if node.Status == "online" && m.tempCollector != nil {
tempCtx, tempCancel := context.WithTimeout(ctx, 30*time.Second) // Increased to accommodate SSH operations via proxy
if node.Status == "online" {
if tempService := m.temperatureService(); tempService != nil && tempService.Enabled() {
tempCtx, tempCancel := context.WithTimeout(ctx, 30*time.Second) // Increased to accommodate SSH operations via proxy
// Determine SSH hostname to use (most robust approach):
// Prefer the resolved host for this node, with cluster overrides when available.
sshHost := modelNode.Host
// Determine SSH hostname to use (most robust approach):
// Prefer the resolved host for this node, with cluster overrides when available.
sshHost := modelNode.Host
if modelNode.IsClusterMember && instanceCfg.IsCluster {
for _, ep := range instanceCfg.ClusterEndpoints {
if strings.EqualFold(ep.NodeName, node.Node) {
if effective := clusterEndpointEffectiveURL(ep); effective != "" {
sshHost = effective
if modelNode.IsClusterMember && instanceCfg.IsCluster {
for _, ep := range instanceCfg.ClusterEndpoints {
if strings.EqualFold(ep.NodeName, node.Node) {
if effective := clusterEndpointEffectiveURL(ep); effective != "" {
sshHost = effective
}
break
}
break
}
}
}
if strings.TrimSpace(sshHost) == "" {
sshHost = node.Node
}
temp, err := m.tempCollector.CollectTemperature(tempCtx, sshHost, node.Node)
tempCancel()
if err == nil && temp != nil && temp.Available {
// Get the current CPU temperature (prefer package, fall back to max)
currentTemp := temp.CPUPackage
if currentTemp == 0 && temp.CPUMax > 0 {
currentTemp = temp.CPUMax
}
// Find previous temperature data for this node to preserve min/max
var prevTemp *models.Temperature
for _, prevNode := range prevInstanceNodes {
if prevNode.ID == modelNode.ID && prevNode.Temperature != nil {
prevTemp = prevNode.Temperature
break
}
}
// Initialize or update min/max tracking
if prevTemp != nil && prevTemp.CPUMin > 0 {
// Preserve existing min/max and update if necessary
temp.CPUMin = prevTemp.CPUMin
temp.CPUMaxRecord = prevTemp.CPUMaxRecord
temp.MinRecorded = prevTemp.MinRecorded
temp.MaxRecorded = prevTemp.MaxRecorded
if strings.TrimSpace(sshHost) == "" {
sshHost = node.Node
}
// Update min if current is lower
if currentTemp > 0 && currentTemp < temp.CPUMin {
temp, err := tempService.Collect(tempCtx, sshHost, node.Node)
tempCancel()
switch {
case err == nil && temp != nil && temp.Available:
// Get the current CPU temperature (prefer package, fall back to max)
currentTemp := temp.CPUPackage
if currentTemp == 0 && temp.CPUMax > 0 {
currentTemp = temp.CPUMax
}
// Find previous temperature data for this node to preserve min/max
var prevTemp *models.Temperature
for _, prevNode := range prevInstanceNodes {
if prevNode.ID == modelNode.ID && prevNode.Temperature != nil {
prevTemp = prevNode.Temperature
break
}
}
// Initialize or update min/max tracking
if prevTemp != nil && prevTemp.CPUMin > 0 {
// Preserve existing min/max and update if necessary
temp.CPUMin = prevTemp.CPUMin
temp.CPUMaxRecord = prevTemp.CPUMaxRecord
temp.MinRecorded = prevTemp.MinRecorded
temp.MaxRecorded = prevTemp.MaxRecorded
// Update min if current is lower
if currentTemp > 0 && currentTemp < temp.CPUMin {
temp.CPUMin = currentTemp
temp.MinRecorded = time.Now()
}
// Update max if current is higher
if currentTemp > temp.CPUMaxRecord {
temp.CPUMaxRecord = currentTemp
temp.MaxRecorded = time.Now()
}
} else if currentTemp > 0 {
// First reading - initialize min/max to current value
temp.CPUMin = currentTemp
temp.MinRecorded = time.Now()
}
// Update max if current is higher
if currentTemp > temp.CPUMaxRecord {
temp.CPUMaxRecord = currentTemp
temp.MinRecorded = time.Now()
temp.MaxRecorded = time.Now()
}
} else if currentTemp > 0 {
// First reading - initialize min/max to current value
temp.CPUMin = currentTemp
temp.CPUMaxRecord = currentTemp
temp.MinRecorded = time.Now()
temp.MaxRecorded = time.Now()
}
modelNode.Temperature = temp
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Float64("cpuPackage", temp.CPUPackage).
Float64("cpuMax", temp.CPUMax).
Float64("cpuMin", temp.CPUMin).
Float64("cpuMaxRecord", temp.CPUMaxRecord).
Int("nvmeCount", len(temp.NVMe)).
Msg("Collected temperature data")
} else if err != nil {
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Bool("isCluster", modelNode.IsClusterMember).
Int("endpointCount", len(instanceCfg.ClusterEndpoints)).
Msg("Temperature collection failed - check SSH access")
} else if temp != nil {
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Bool("available", temp.Available).
Msg("Temperature data unavailable after collection")
modelNode.Temperature = temp
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Float64("cpuPackage", temp.CPUPackage).
Float64("cpuMax", temp.CPUMax).
Float64("cpuMin", temp.CPUMin).
Float64("cpuMaxRecord", temp.CPUMaxRecord).
Int("nvmeCount", len(temp.NVMe)).
Msg("Collected temperature data")
case err != nil:
if !stderrors.Is(err, ErrTemperatureMonitoringDisabled) && !stderrors.Is(err, ErrTemperatureCollectorUnavailable) {
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Bool("isCluster", modelNode.IsClusterMember).
Int("endpointCount", len(instanceCfg.ClusterEndpoints)).
Err(err).
Msg("Temperature collection failed - check SSH access")
}
case temp != nil:
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Bool("available", temp.Available).
Msg("Temperature data unavailable after collection")
}
}
}

View file

@ -0,0 +1,43 @@
package monitoring
import (
"path/filepath"
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
)
func TestMonitorTemperatureCollectorToggle(t *testing.T) {
t.Parallel()
cfg := &config.Config{TemperatureMonitoringEnabled: false}
service := newTemperatureService(false, "root", filepath.Join(t.TempDir(), "id_ed25519_sensors"))
m := &Monitor{
config: cfg,
tempService: service,
}
if provider := m.temperatureService(); provider == nil {
t.Fatalf("expected temperature service to be present")
} else if provider.Enabled() {
t.Fatalf("expected service to start disabled")
}
m.EnableTemperatureMonitoring()
if !cfg.TemperatureMonitoringEnabled {
t.Fatalf("expected config flag to be true after enabling")
}
if provider := m.temperatureService(); provider == nil || !provider.Enabled() {
t.Fatalf("expected service to be enabled after EnableTemperatureMonitoring")
}
m.DisableTemperatureMonitoring()
if cfg.TemperatureMonitoringEnabled {
t.Fatalf("expected config flag to be false after disabling")
}
if provider := m.temperatureService(); provider == nil || provider.Enabled() {
t.Fatalf("expected service to be disabled after DisableTemperatureMonitoring")
}
}

View file

@ -0,0 +1,99 @@
package monitoring
import (
"context"
stderrors "errors"
"sync"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
var (
// ErrTemperatureMonitoringDisabled indicates that temperature polling is disabled globally.
ErrTemperatureMonitoringDisabled = stderrors.New("temperature monitoring disabled")
// ErrTemperatureCollectorUnavailable indicates the collector could not be created or is misconfigured.
ErrTemperatureCollectorUnavailable = stderrors.New("temperature collector unavailable")
)
// TemperatureService defines the contract used by the monitor to collect temperature data.
type TemperatureService interface {
Enabled() bool
Collect(ctx context.Context, host, nodeName string) (*models.Temperature, error)
Enable()
Disable()
}
type temperatureService struct {
mu sync.RWMutex
enabled bool
user string
keyPath string
collector *TemperatureCollector
}
func newTemperatureService(enabled bool, user, keyPath string) TemperatureService {
return &temperatureService{
enabled: enabled,
user: user,
keyPath: keyPath,
}
}
func (s *temperatureService) Enabled() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.enabled
}
func (s *temperatureService) Enable() {
s.mu.Lock()
defer s.mu.Unlock()
if s.enabled {
return
}
s.enabled = true
if s.collector == nil && s.user != "" && s.keyPath != "" {
s.collector = NewTemperatureCollector(s.user, s.keyPath)
}
}
func (s *temperatureService) Disable() {
s.mu.Lock()
defer s.mu.Unlock()
s.enabled = false
s.collector = nil
}
func (s *temperatureService) Collect(ctx context.Context, host, nodeName string) (*models.Temperature, error) {
s.mu.RLock()
enabled := s.enabled
collector := s.collector
s.mu.RUnlock()
if !enabled {
return nil, ErrTemperatureMonitoringDisabled
}
if collector == nil {
s.mu.Lock()
if s.enabled && s.collector == nil && s.user != "" && s.keyPath != "" {
s.collector = NewTemperatureCollector(s.user, s.keyPath)
}
collector = s.collector
enabled = s.enabled
s.mu.Unlock()
if !enabled {
return nil, ErrTemperatureMonitoringDisabled
}
}
if collector == nil {
return nil, ErrTemperatureCollectorUnavailable
}
return collector.CollectTemperature(ctx, host, nodeName)
}