From dbac8632b77c6e62f25a2ea58220653ef6c1b9b7 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sat, 13 Dec 2025 14:18:05 +0000 Subject: [PATCH] feat: add persistent metrics history API endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add GET /api/metrics-store/history endpoint for querying SQLite-backed metrics - Support flexible time ranges: 1h, 6h, 12h, 24h, 7d, 30d, 90d - Return aggregated data with min/max values for longer time ranges - Add TypeScript types and ChartsAPI.getMetricsHistory() client method This enables frontend charts to visualize long-term trends using the tiered retention system (raw → minute → hourly → daily averages). --- .agent/artifacts/monetization_roadmap.md | 152 ----------------------- frontend-modern/src/api/charts.ts | 83 +++++++++++++ internal/api/router.go | 142 +++++++++++++++++++++ 3 files changed, 225 insertions(+), 152 deletions(-) delete mode 100644 .agent/artifacts/monetization_roadmap.md diff --git a/.agent/artifacts/monetization_roadmap.md b/.agent/artifacts/monetization_roadmap.md deleted file mode 100644 index 816d1c3..0000000 --- a/.agent/artifacts/monetization_roadmap.md +++ /dev/null @@ -1,152 +0,0 @@ -# Pulse Monetization Readiness Roadmap - -> Created: 2025-12-13 -> Status: Planning → Implementation - -## Executive Summary - -Based on GitHub insights (3,068 stars, 994 cloners/2wks, 458k downloads latest release) and user feedback, Pulse has strong organic adoption. The codebase already has solid foundations for monetization, but several enhancements will make tier-based pricing natural. - -## Current State ✅ - -### What's Already Built -- **Historical metrics storage** (SQLite with tiered rollups: raw→minute→hourly→daily) -- **Unified resource model** (platform-agnostic Resource type) -- **Multi-platform support** (PVE, PBS, PMG, Docker, K8s, TrueNAS, Host-Agent) -- **Cross-source deduplication** (ResourceIdentity) -- **AI opt-in with BYO key** (no cost burden on Pulse) -- **Cluster awareness** (ClusterEndpoints, multi-node) - -### Default Retention (hardcoded in `internal/metrics/store.go`) -- Raw: 2 hours -- Minute: 24 hours -- Hourly: 7 days -- Daily: 90 days - ---- - -## Phase 1: Configurable Foundations (This Sprint) - -### 1.1 Configurable Metrics Retention ✅ COMPLETE -**Goal:** Allow users to configure retention periods via settings. - -**Why it matters for monetization:** -- Natural tier split: "Free = 7 days, Paid = 90 days" -- Users who want longer history have a clear upgrade path - -**Implementation (Completed 2025-12-13):** -- [x] Added `MetricsRetention*` fields to `SystemSettings` in `config/persistence.go` -- [x] Added corresponding fields to `Config` in `config/config.go` -- [x] Set sensible defaults (Raw: 2h, Minute: 24h, Hourly: 7d, Daily: 90d) -- [x] Wired config to `metrics.StoreConfig` in `monitor.go` -- [x] Added debug logging showing active retention values - -**Files modified:** -- `internal/config/persistence.go` - Added 4 new fields to SystemSettings -- `internal/config/config.go` - Added 4 new fields to Config, loading logic, defaults -- `internal/monitoring/monitor.go` - Wire config values to metrics store - -**Next steps for this feature:** -- [ ] Add retention settings UI in Settings page -- [ ] Expose via API for programmatic configuration - -### 1.2 Metrics History API Endpoint -**Goal:** Expose historical metrics via REST API for frontend charts. - -**Why it matters:** -- Users can see value of stored history -- Charts showing 7d/30d/90d trends become possible -- Makes the retention config visible to users - -**Implementation:** -- [ ] Add `/api/metrics/history/:resourceType/:resourceId` endpoint -- [ ] Support query params: `start`, `end`, `metric` (cpu, memory, disk, network) -- [ ] Auto-select appropriate tier based on time range - ---- - -## Phase 2: Multi-Tenant Foundations (Next Sprint) - -### 2.1 Tenant/Organization Model -**Goal:** Add optional tenant isolation for MSP use cases. - -**Why it matters:** -- MSPs are the primary paying audience -- Each client = one tenant, isolated data -- Per-tenant billing becomes possible - -**Implementation:** -- [ ] Add `TenantID` field to Resource model (optional, empty = single-tenant) -- [ ] Add tenant filtering to Store queries -- [ ] Add tenant selector to UI (hidden in single-tenant mode) - -### 2.2 RBAC Enhancements -**Goal:** Role-based access control per tenant. - -**Why it matters:** -- Enterprise requirement -- MSP operators vs client viewers - ---- - -## Phase 3: Reports & Exports (Future) - -### 3.1 Backup Compliance Reports -**Goal:** Weekly/monthly PDF/email reports on backup status. - -**Why it matters:** -- Top user request ("List of unbacked-up VMs") -- Obvious paid feature -- Sticky habit (scheduled reports) - -### 3.2 Capacity Planning Reports -**Goal:** Trend-based forecasting for storage and compute. - -**Why it matters:** -- Uses historical data (paid tier dependency) -- High value for ops teams - ---- - -## Phase 4: AI Enhancements (Future) - -### 4.1 Grounded Incident Summaries -**Goal:** "Explain this alert with evidence" + "Draft incident report" - -**Why it matters:** -- Only valuable because of Pulse context -- Can't be replicated by Claude Code alone - -### 4.2 What-Changed Analysis -**Goal:** "What changed in the last 24h that might explain this issue?" - -**Why it matters:** -- Requires historical data (ties to paid retention) -- High-value for triage - ---- - -## Proposed Tier Structure (Future Reference) - -| Feature | Free | Pro | Enterprise | -|---------|------|-----|------------| -| Clusters | 1 | Unlimited | Unlimited | -| Retention | 7 days | 90 days | 1 year | -| Alert channels | 1 | 5 | Unlimited | -| Backup reports | - | Weekly | Custom | -| RBAC | - | - | ✓ | -| Multi-tenant | - | - | ✓ | -| AI features | Basic | Full | Full + Custom | -| Support | Community | Email | Priority | - ---- - -## Implementation Priority - -1. **Configurable retention** (enables tier split) -2. **Metrics history API** (makes history visible) -3. **Frontend history charts** (user-visible value) -4. **Multi-tenant model** (MSP readiness) -5. **Reports** (paid feature) - -Starting with #1 now. diff --git a/frontend-modern/src/api/charts.ts b/frontend-modern/src/api/charts.ts index 9a8f23f..0c0dd9e 100644 --- a/frontend-modern/src/api/charts.ts +++ b/frontend-modern/src/api/charts.ts @@ -13,6 +13,14 @@ export interface MetricPoint { value: number; } +// Extended metric point with min/max for aggregated data +export interface AggregatedMetricPoint { + timestamp: number; // Unix timestamp in milliseconds + value: number; + min: number; + max: number; +} + export interface ChartData { cpu?: MetricPoint[]; memory?: MetricPoint[]; @@ -38,6 +46,52 @@ export interface ChartsResponse { stats: ChartStats; } +// Persistent metrics history types (SQLite-backed, longer retention) +export type HistoryTimeRange = '1h' | '6h' | '12h' | '24h' | '7d' | '30d' | '90d'; +export type ResourceType = 'node' | 'guest' | 'storage' | 'docker' | 'dockerHost'; + +export interface MetricsHistoryParams { + resourceType: ResourceType; + resourceId: string; + metric?: string; // Optional: 'cpu', 'memory', 'disk', etc. Omit for all metrics + range?: HistoryTimeRange; // Default: '24h' +} + +export interface SingleMetricHistoryResponse { + resourceType: string; + resourceId: string; + metric: string; + range: string; + start: number; // Unix timestamp in milliseconds + end: number; // Unix timestamp in milliseconds + points: AggregatedMetricPoint[]; +} + +export interface AllMetricsHistoryResponse { + resourceType: string; + resourceId: string; + range: string; + start: number; // Unix timestamp in milliseconds + end: number; // Unix timestamp in milliseconds + metrics: Record; +} + +export interface MetricsStoreStats { + enabled: boolean; + dbPath?: string; + dbSize?: number; + rawCount?: number; + minuteCount?: number; + hourlyCount?: number; + dailyCount?: number; + totalWrites?: number; + bufferSize?: number; + lastFlush?: string; + lastRollup?: string; + lastRetention?: string; + error?: string; +} + export type TimeRange = '5m' | '15m' | '30m' | '1h' | '4h' | '12h' | '24h' | '7d'; export class ChartsAPI { @@ -65,4 +119,33 @@ export class ChartsAPI { const url = `${this.baseUrl}/storage/charts?range=${rangeMinutes}`; return apiFetchJSON(url); } + + /** + * Fetch persistent metrics history for a specific resource + * This uses the SQLite-backed store with longer retention (up to 90 days) + * @param params Query parameters + */ + static async getMetricsHistory(params: MetricsHistoryParams): Promise { + const searchParams = new URLSearchParams({ + resourceType: params.resourceType, + resourceId: params.resourceId, + }); + if (params.metric) { + searchParams.set('metric', params.metric); + } + if (params.range) { + searchParams.set('range', params.range); + } + const url = `${this.baseUrl}/metrics-store/history?${searchParams.toString()}`; + return apiFetchJSON(url); + } + + /** + * Fetch statistics about the persistent metrics store + */ + static async getMetricsStoreStats(): Promise { + const url = `${this.baseUrl}/metrics-store/stats`; + return apiFetchJSON(url); + } } + diff --git a/internal/api/router.go b/internal/api/router.go index 39c8c76..f47d92f 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -216,6 +216,7 @@ func (r *Router) setupRoutes() { r.mux.HandleFunc("/api/storage-charts", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleStorageCharts))) r.mux.HandleFunc("/api/charts", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleCharts))) r.mux.HandleFunc("/api/metrics-store/stats", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleMetricsStoreStats))) + r.mux.HandleFunc("/api/metrics-store/history", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleMetricsHistory))) r.mux.HandleFunc("/api/diagnostics", RequireAuth(r.config, r.handleDiagnostics)) r.mux.HandleFunc("/api/diagnostics/temperature-proxy/register-nodes", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, r.handleDiagnosticsRegisterProxyNodes))) r.mux.HandleFunc("/api/diagnostics/docker/prepare-token", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, r.handleDiagnosticsDockerPrepareToken))) @@ -3473,6 +3474,147 @@ func (r *Router) handleMetricsStoreStats(w http.ResponseWriter, req *http.Reques } } +// handleMetricsHistory returns historical metrics from the persistent SQLite store +// Query params: +// - resourceType: "node", "guest", "storage", "docker", "dockerHost" (required) +// - resourceId: the resource identifier (required) +// - metric: "cpu", "memory", "disk", etc. (optional, omit for all metrics) +// - range: time range like "1h", "24h", "7d", "30d", "90d" (optional, default "24h") +func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + store := r.monitor.GetMetricsStore() + if store == nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]interface{}{ + "error": "Persistent metrics store not available", + }) + return + } + + query := req.URL.Query() + resourceType := query.Get("resourceType") + resourceID := query.Get("resourceId") + metricType := query.Get("metric") + timeRange := query.Get("range") + + if resourceType == "" || resourceID == "" { + http.Error(w, "resourceType and resourceId are required", http.StatusBadRequest) + return + } + + // Parse time range + var duration time.Duration + switch timeRange { + case "1h": + duration = time.Hour + case "6h": + duration = 6 * time.Hour + case "12h": + duration = 12 * time.Hour + case "24h", "1d", "": + duration = 24 * time.Hour + case "7d": + duration = 7 * 24 * time.Hour + case "30d": + duration = 30 * 24 * time.Hour + case "90d": + duration = 90 * 24 * time.Hour + default: + // Try parsing as duration + var err error + duration, err = time.ParseDuration(timeRange) + if err != nil { + duration = 24 * time.Hour // Default to 24 hours + } + } + + end := time.Now() + start := end.Add(-duration) + + var response interface{} + + if metricType != "" { + // Query single metric type + points, err := store.Query(resourceType, resourceID, metricType, start, end) + if err != nil { + log.Error().Err(err). + Str("resourceType", resourceType). + Str("resourceId", resourceID). + Str("metric", metricType). + Msg("Failed to query metrics history") + http.Error(w, "Failed to query metrics", http.StatusInternalServerError) + return + } + + // Convert to frontend format (timestamps in milliseconds) + apiPoints := make([]map[string]interface{}, len(points)) + for i, p := range points { + apiPoints[i] = map[string]interface{}{ + "timestamp": p.Timestamp.UnixMilli(), + "value": p.Value, + "min": p.Min, + "max": p.Max, + } + } + + response = map[string]interface{}{ + "resourceType": resourceType, + "resourceId": resourceID, + "metric": metricType, + "range": timeRange, + "start": start.UnixMilli(), + "end": end.UnixMilli(), + "points": apiPoints, + } + } else { + // Query all metrics for this resource + metricsMap, err := store.QueryAll(resourceType, resourceID, start, end) + if err != nil { + log.Error().Err(err). + Str("resourceType", resourceType). + Str("resourceId", resourceID). + Msg("Failed to query all metrics history") + http.Error(w, "Failed to query metrics", http.StatusInternalServerError) + return + } + + // Convert to frontend format + apiData := make(map[string][]map[string]interface{}) + for metric, points := range metricsMap { + apiPoints := make([]map[string]interface{}, len(points)) + for i, p := range points { + apiPoints[i] = map[string]interface{}{ + "timestamp": p.Timestamp.UnixMilli(), + "value": p.Value, + "min": p.Min, + "max": p.Max, + } + } + apiData[metric] = apiPoints + } + + response = map[string]interface{}{ + "resourceType": resourceType, + "resourceId": resourceID, + "range": timeRange, + "start": start.UnixMilli(), + "end": end.UnixMilli(), + "metrics": apiData, + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + log.Error().Err(err).Msg("Failed to encode metrics history response") + http.Error(w, "Internal server error", http.StatusInternalServerError) + } +} + // handleConfig handles configuration requests func (r *Router) handleConfig(w http.ResponseWriter, req *http.Request) { if req.Method != http.MethodGet {