From 0a9f92753b0089f9e3d2c91f39ad7786419bba44 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sat, 13 Dec 2025 14:11:31 +0000 Subject: [PATCH] feat: add configurable metrics retention settings - Add MetricsRetentionRawHours, MetricsRetentionMinuteHours, MetricsRetentionHourlyDays, MetricsRetentionDailyDays to SystemSettings - Wire settings from system.json through Config to metrics store initialization - Set sensible defaults: Raw=2h, Minute=24h, Hourly=7d, Daily=90d - Log active retention values on startup for transparency Users can now customize how long metrics are stored at each aggregation tier. --- .agent/artifacts/monetization_roadmap.md | 152 +++++++++++++++++++++++ internal/config/config.go | 31 ++++- internal/config/persistence.go | 9 ++ internal/monitoring/monitor.go | 23 +++- 4 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 .agent/artifacts/monetization_roadmap.md diff --git a/.agent/artifacts/monetization_roadmap.md b/.agent/artifacts/monetization_roadmap.md new file mode 100644 index 0000000..816d1c3 --- /dev/null +++ b/.agent/artifacts/monetization_roadmap.md @@ -0,0 +1,152 @@ +# Pulse Monetization Readiness Roadmap + +> Created: 2025-12-13 +> Status: Planning → Implementation + +## Executive Summary + +Based on GitHub insights (3,068 stars, 994 cloners/2wks, 458k downloads latest release) and user feedback, Pulse has strong organic adoption. The codebase already has solid foundations for monetization, but several enhancements will make tier-based pricing natural. + +## Current State ✅ + +### What's Already Built +- **Historical metrics storage** (SQLite with tiered rollups: raw→minute→hourly→daily) +- **Unified resource model** (platform-agnostic Resource type) +- **Multi-platform support** (PVE, PBS, PMG, Docker, K8s, TrueNAS, Host-Agent) +- **Cross-source deduplication** (ResourceIdentity) +- **AI opt-in with BYO key** (no cost burden on Pulse) +- **Cluster awareness** (ClusterEndpoints, multi-node) + +### Default Retention (hardcoded in `internal/metrics/store.go`) +- Raw: 2 hours +- Minute: 24 hours +- Hourly: 7 days +- Daily: 90 days + +--- + +## Phase 1: Configurable Foundations (This Sprint) + +### 1.1 Configurable Metrics Retention ✅ COMPLETE +**Goal:** Allow users to configure retention periods via settings. + +**Why it matters for monetization:** +- Natural tier split: "Free = 7 days, Paid = 90 days" +- Users who want longer history have a clear upgrade path + +**Implementation (Completed 2025-12-13):** +- [x] Added `MetricsRetention*` fields to `SystemSettings` in `config/persistence.go` +- [x] Added corresponding fields to `Config` in `config/config.go` +- [x] Set sensible defaults (Raw: 2h, Minute: 24h, Hourly: 7d, Daily: 90d) +- [x] Wired config to `metrics.StoreConfig` in `monitor.go` +- [x] Added debug logging showing active retention values + +**Files modified:** +- `internal/config/persistence.go` - Added 4 new fields to SystemSettings +- `internal/config/config.go` - Added 4 new fields to Config, loading logic, defaults +- `internal/monitoring/monitor.go` - Wire config values to metrics store + +**Next steps for this feature:** +- [ ] Add retention settings UI in Settings page +- [ ] Expose via API for programmatic configuration + +### 1.2 Metrics History API Endpoint +**Goal:** Expose historical metrics via REST API for frontend charts. + +**Why it matters:** +- Users can see value of stored history +- Charts showing 7d/30d/90d trends become possible +- Makes the retention config visible to users + +**Implementation:** +- [ ] Add `/api/metrics/history/:resourceType/:resourceId` endpoint +- [ ] Support query params: `start`, `end`, `metric` (cpu, memory, disk, network) +- [ ] Auto-select appropriate tier based on time range + +--- + +## Phase 2: Multi-Tenant Foundations (Next Sprint) + +### 2.1 Tenant/Organization Model +**Goal:** Add optional tenant isolation for MSP use cases. + +**Why it matters:** +- MSPs are the primary paying audience +- Each client = one tenant, isolated data +- Per-tenant billing becomes possible + +**Implementation:** +- [ ] Add `TenantID` field to Resource model (optional, empty = single-tenant) +- [ ] Add tenant filtering to Store queries +- [ ] Add tenant selector to UI (hidden in single-tenant mode) + +### 2.2 RBAC Enhancements +**Goal:** Role-based access control per tenant. + +**Why it matters:** +- Enterprise requirement +- MSP operators vs client viewers + +--- + +## Phase 3: Reports & Exports (Future) + +### 3.1 Backup Compliance Reports +**Goal:** Weekly/monthly PDF/email reports on backup status. + +**Why it matters:** +- Top user request ("List of unbacked-up VMs") +- Obvious paid feature +- Sticky habit (scheduled reports) + +### 3.2 Capacity Planning Reports +**Goal:** Trend-based forecasting for storage and compute. + +**Why it matters:** +- Uses historical data (paid tier dependency) +- High value for ops teams + +--- + +## Phase 4: AI Enhancements (Future) + +### 4.1 Grounded Incident Summaries +**Goal:** "Explain this alert with evidence" + "Draft incident report" + +**Why it matters:** +- Only valuable because of Pulse context +- Can't be replicated by Claude Code alone + +### 4.2 What-Changed Analysis +**Goal:** "What changed in the last 24h that might explain this issue?" + +**Why it matters:** +- Requires historical data (ties to paid retention) +- High-value for triage + +--- + +## Proposed Tier Structure (Future Reference) + +| Feature | Free | Pro | Enterprise | +|---------|------|-----|------------| +| Clusters | 1 | Unlimited | Unlimited | +| Retention | 7 days | 90 days | 1 year | +| Alert channels | 1 | 5 | Unlimited | +| Backup reports | - | Weekly | Custom | +| RBAC | - | - | ✓ | +| Multi-tenant | - | - | ✓ | +| AI features | Basic | Full | Full + Custom | +| Support | Community | Email | Priority | + +--- + +## Implementation Priority + +1. **Configurable retention** (enables tier split) +2. **Metrics history API** (makes history visible) +3. **Frontend history charts** (user-visible value) +4. **Multi-tenant model** (MSP readiness) +5. **Reports** (paid feature) + +Starting with #1 now. diff --git a/internal/config/config.go b/internal/config/config.go index 8910bd2..92bb8fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -95,7 +95,6 @@ type Config struct { PMGPollingInterval time.Duration `envconfig:"PMG_POLLING_INTERVAL"` // PMG polling interval (60s default) ConcurrentPolling bool `envconfig:"CONCURRENT_POLLING" default:"true"` ConnectionTimeout time.Duration `envconfig:"CONNECTION_TIMEOUT" default:"45s"` // Increased for slow storage operations - MetricsRetentionDays int `envconfig:"METRICS_RETENTION_DAYS" default:"7"` BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"` BackupPollingInterval time.Duration `envconfig:"BACKUP_POLLING_INTERVAL"` EnableBackupPolling bool `envconfig:"ENABLE_BACKUP_POLLING" default:"true"` @@ -112,6 +111,14 @@ type Config struct { DNSCacheTimeout time.Duration `envconfig:"DNS_CACHE_TIMEOUT" default:"5m" json:"dnsCacheTimeout"` SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring + // Metrics retention settings (tiered storage) + // These control how long historical metrics are retained at each aggregation level. + MetricsRetentionRawHours int `json:"metricsRetentionRawHours"` // Raw data (~5s intervals), default: 2 hours + MetricsRetentionMinuteHours int `json:"metricsRetentionMinuteHours"` // Minute averages, default: 24 hours + MetricsRetentionHourlyDays int `json:"metricsRetentionHourlyDays"` // Hourly averages, default: 7 days + MetricsRetentionDailyDays int `json:"metricsRetentionDailyDays"` // Daily averages, default: 90 days + + // Logging settings LogLevel string `envconfig:"LOG_LEVEL" default:"info"` LogFormat string `envconfig:"LOG_FORMAT" default:"auto"` // "json", "console", or "auto" @@ -547,7 +554,6 @@ func Load() (*Config, error) { AppRoot: detectAppRoot(), ConcurrentPolling: true, ConnectionTimeout: 60 * time.Second, - MetricsRetentionDays: 7, BackupPollingCycles: 10, BackupPollingInterval: 0, EnableBackupPolling: true, @@ -576,6 +582,11 @@ func Load() (*Config, error) { TemperatureMonitoringEnabled: true, EnvOverrides: make(map[string]bool), OIDC: NewOIDCConfig(), + // Metrics retention defaults (tiered) + MetricsRetentionRawHours: 2, // 2 hours of raw ~5s data + MetricsRetentionMinuteHours: 24, // 24 hours of minute averages + MetricsRetentionHourlyDays: 7, // 7 days of hourly averages + MetricsRetentionDailyDays: 90, // 90 days of daily averages } cfg.Discovery = DefaultDiscoveryConfig() @@ -671,11 +682,27 @@ func Load() (*Config, error) { } // Load HideLocalLogin cfg.HideLocalLogin = systemSettings.HideLocalLogin + + // Load metrics retention settings (only override if explicitly set) + if systemSettings.MetricsRetentionRawHours > 0 { + cfg.MetricsRetentionRawHours = systemSettings.MetricsRetentionRawHours + } + if systemSettings.MetricsRetentionMinuteHours > 0 { + cfg.MetricsRetentionMinuteHours = systemSettings.MetricsRetentionMinuteHours + } + if systemSettings.MetricsRetentionHourlyDays > 0 { + cfg.MetricsRetentionHourlyDays = systemSettings.MetricsRetentionHourlyDays + } + if systemSettings.MetricsRetentionDailyDays > 0 { + cfg.MetricsRetentionDailyDays = systemSettings.MetricsRetentionDailyDays + } + // APIToken no longer loaded from system.json - only from .env log.Info(). Str("updateChannel", cfg.UpdateChannel). Str("logLevel", cfg.LogLevel). Dur("dnsCacheTimeout", cfg.DNSCacheTimeout). + Int("metricsRetentionDailyDays", cfg.MetricsRetentionDailyDays). Msg("Loaded system configuration") } else { // No system.json exists - create default one diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 327d552..3fe77ba 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -845,6 +845,15 @@ type SystemSettings struct { SSHPort int `json:"sshPort,omitempty"` // Default SSH port for temperature monitoring (0 = use 22) WebhookAllowedPrivateCIDRs string `json:"webhookAllowedPrivateCIDRs,omitempty"` // Comma-separated list of private CIDR ranges allowed for webhooks (e.g., "192.168.1.0/24,10.0.0.0/8") HideLocalLogin bool `json:"hideLocalLogin"` // Hide local login form (username/password) + + // Metrics retention configuration (in hours) + // These control how long historical metrics are stored at each aggregation tier. + // Longer retention enables trend analysis but increases storage usage. + MetricsRetentionRawHours int `json:"metricsRetentionRawHours,omitempty"` // Raw data (~5s intervals), default: 2 hours + MetricsRetentionMinuteHours int `json:"metricsRetentionMinuteHours,omitempty"` // Minute averages, default: 24 hours + MetricsRetentionHourlyDays int `json:"metricsRetentionHourlyDays,omitempty"` // Hourly averages, default: 7 days + MetricsRetentionDailyDays int `json:"metricsRetentionDailyDays,omitempty"` // Daily averages, default: 90 days + // APIToken removed - now handled via .env file only } diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index f195740..f94e8a7 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -2830,15 +2830,34 @@ func New(cfg *config.Config) (*Monitor, error) { guestAgentVersionTimeout := parseDurationEnv("GUEST_AGENT_VERSION_TIMEOUT", defaultGuestAgentVersionTimeout) guestAgentRetries := parseIntEnv("GUEST_AGENT_RETRIES", defaultGuestAgentRetries) - // Initialize persistent metrics store (SQLite) + // Initialize persistent metrics store (SQLite) with configurable retention var metricsStore *metrics.Store metricsStoreConfig := metrics.DefaultConfig(cfg.DataPath) + // Override retention settings from config (allows tier-based pricing in future) + if cfg.MetricsRetentionRawHours > 0 { + metricsStoreConfig.RetentionRaw = time.Duration(cfg.MetricsRetentionRawHours) * time.Hour + } + if cfg.MetricsRetentionMinuteHours > 0 { + metricsStoreConfig.RetentionMinute = time.Duration(cfg.MetricsRetentionMinuteHours) * time.Hour + } + if cfg.MetricsRetentionHourlyDays > 0 { + metricsStoreConfig.RetentionHourly = time.Duration(cfg.MetricsRetentionHourlyDays) * 24 * time.Hour + } + if cfg.MetricsRetentionDailyDays > 0 { + metricsStoreConfig.RetentionDaily = time.Duration(cfg.MetricsRetentionDailyDays) * 24 * time.Hour + } ms, err := metrics.NewStore(metricsStoreConfig) if err != nil { log.Error().Err(err).Msg("Failed to initialize persistent metrics store - continuing with in-memory only") } else { metricsStore = ms - log.Info().Str("path", metricsStoreConfig.DBPath).Msg("Persistent metrics store initialized") + log.Info(). + Str("path", metricsStoreConfig.DBPath). + Dur("retentionRaw", metricsStoreConfig.RetentionRaw). + Dur("retentionMinute", metricsStoreConfig.RetentionMinute). + Dur("retentionHourly", metricsStoreConfig.RetentionHourly). + Dur("retentionDaily", metricsStoreConfig.RetentionDaily). + Msg("Persistent metrics store initialized with configurable retention") } m := &Monitor{