{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 10, "links": [ { "asDropdown": false, "icon": "question", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": false, "title": "Help", "tooltip": "", "type": "link", "url": "https://shb.skarabox.com/blocks-monitoring.html" } ], "panels": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "green", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "% failed" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "color-background" } }, { "id": "color", "value": { "mode": "continuous-GrYlRd" } }, { "id": "max", "value": 1 } ] }, { "matcher": { "id": "byName", "options": "total" }, "properties": [ { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "color-background" } }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "transparent", "value": 1 } ] } } ] }, { "matcher": { "id": "byType", "options": "string" }, "properties": [ { "id": "custom.minWidth", "value": 150 } ] }, { "matcher": { "id": "byType", "options": "number" }, "properties": [ { "id": "custom.width", "value": 100 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "id": 11, "options": { "cellHeight": "sm", "enablePagination": true, "frozenColumns": {}, "showHeader": true, "sortBy": [] }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "exemplar": false, "expr": "increase(systemd_unit_state{name=~\"[[job]].service\", state=\"activating\"}[7d])", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "exemplar": false, "expr": "increase(systemd_unit_state{name=~\"[[job]].service\", state=\"failed\"}[7d])", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "B" } ], "title": "Backup Jobs in the Past Week", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "merge", "options": {} }, { "id": "groupingToMatrix", "options": { "columnField": "state", "rowField": "name", "valueField": "Value" } }, { "id": "calculateField", "options": { "alias": "total", "binary": { "left": { "matcher": { "id": "byName", "options": "activating" } }, "operator": "+", "right": { "matcher": { "id": "byName", "options": "failed" } } }, "mode": "binary", "reduce": { "include": [ "activating", "failed" ], "reducer": "sum" } } }, { "id": "calculateField", "options": { "alias": "% failed", "binary": { "left": { "matcher": { "id": "byName", "options": "failed" } }, "operator": "/", "right": { "matcher": { "id": "byName", "options": "total" } } }, "mode": "binary", "reduce": { "reducer": "sum" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "total" } ] } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "failed" } ] } }, { "id": "organize", "options": { "excludeByName": {}, "includeByName": {}, "indexByName": {}, "renameByName": { "activating": "success", "name\\state": "Job" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 } ] }, "unit": "dateTimeFromNow" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "id": 15, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true, "width": 300 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "editorMode": "code", "exemplar": false, "expr": "systemd_timer_next_trigger_seconds{name=~\"$job.timer\"} * 1000", "format": "time_series", "instant": false, "legendFormat": "{{name}}", "range": true, "refId": "A" } ], "title": "Schedule", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "fixed" }, "custom": { "axisPlacement": "auto", "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [ { "options": { "1": { "color": "green", "index": 0, "text": "Running" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "#EAB839", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, "id": 13, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "mergeValues": true, "rowHeight": 0.9, "showValue": "never", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "exemplar": false, "expr": "max(systemd_unit_state{name=~\"$job.service\", state=\"activating\"} > 0 or on(name) label_replace(clamp(systemd_timer_last_trigger_seconds{name=~\"$job.timer\"} - (systemd_timer_last_trigger_seconds{name=~\"$job.timer\"} offset 1s) > 0, 0, 1), \"name\", \"$1.service\", \"name\", \"(.*).timer\")) by (name)", "format": "time_series", "instant": false, "key": "Q-e1d5c07a-8dcc-4f34-aa5c-cdebcbdda322-0", "legendFormat": "{{name}}", "range": true, "refId": "A" } ], "title": "Backups Jobs", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "id": 1, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "expr": "rate(node_disk_io_time_seconds_total{device=~\"sd.*\"}[2m])", "legendFormat": "{{device}}", "range": true, "refId": "A" } ], "title": "Disk IO Time", "type": "timeseries" }, { "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 12, "x": 12, "y": 17 }, "id": 16, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "If the log panel is empty, it may be because the amount of lines is too high.T ry filtering a few jobs first.", "mode": "markdown" }, "pluginVersion": "12.2.0", "title": "", "type": "text" }, { "datasource": { "default": false, "type": "loki", "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 30, "w": 12, "x": 12, "y": 19 }, "id": 3, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, "enableLogDetails": false, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": false }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "loki", "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "direction": "backward", "editorMode": "code", "expr": "{unit=~\"$job.*.service\"}", "queryType": "range", "refId": "A" } ], "title": "Logs - $job", "type": "logs" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "dashed" } }, "decimals": 2, "fieldMinMax": false, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "rel" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "percentunit" }, { "id": "max", "value": 1.1 } ] }, { "matcher": { "id": "byFrameRefID", "options": "abs" }, "properties": [] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 8, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "expr": "1 - node_filesystem_avail_bytes{device!~\"ramfs|tmpfs|none\", mountpoint=~\"$mountpoints\"} / node_filesystem_size_bytes{device!~\"ramfs|tmpfs|none\", mountpoint=~\"$mountpoints\"}", "hide": true, "legendFormat": "{{mountpoint}}", "range": true, "refId": "rel" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "expr": "node_filesystem_size_bytes{device!~\"ramfs|tmpfs|none\", mountpoint=~\"$mountpoints\"} - node_filesystem_avail_bytes{device!~\"ramfs|tmpfs|none\", mountpoint=~\"$mountpoints\"}", "hide": false, "instant": false, "legendFormat": "{{mountpoint}}", "range": true, "refId": "abs" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "expr": "node_filesystem_size_bytes{device!~\"ramfs|tmpfs|none\", mountpoint=~\"$mountpoints\"}", "hide": false, "instant": false, "legendFormat": "{{mountpoint}}", "range": true, "refId": "max" } ], "title": "Disk Usage", "transformations": [ { "id": "calculateField", "options": { "alias": "max", "binary": { "left": { "matcher": { "id": "byName", "options": "/srv/backup" } }, "operator": "*", "right": { "fixed": "1.1" } }, "mode": "binary", "reduce": { "reducer": "sum" }, "replaceFields": false, "unary": { "fieldName": "/srv/backup", "operator": "percent" } } }, { "id": "configFromData", "options": { "applyTo": { "id": "byFrameRefID", "options": "abs" }, "configRefId": "max", "mappings": [ { "fieldName": "/srv/backup", "handlerArguments": { "threshold": { "color": "red" } }, "handlerKey": "threshold1" }, { "fieldName": "max", "handlerKey": "max" } ] } }, { "id": "organize", "options": { "excludeByName": { "max": true }, "includeByName": {}, "indexByName": {}, "renameByName": {} } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "id": 5, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", "expr": "avg(rate(node_cpu_seconds_total{mode!=\"idle\"}[2m])) by (instance, mode)", "legendFormat": "{{instance}} -- {{mode}}", "range": true, "refId": "A" } ], "title": "CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": 3600000, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "dashed+area" } }, "decimals": 2, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "transparent", "value": 0.05 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "perc" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "max", "value": 1 }, { "id": "unit", "value": "percentunit" }, { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": false } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "id": 14, "options": { "legend": { "calcs": [ "max", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg by(instance) (node_memory_MemAvailable_bytes)", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "legendFormat": "{{instance}} - total", "range": true, "refId": "available", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg by(instance) (node_memory_MemTotal_bytes)", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "instant": false, "legendFormat": "{{instance}} - available", "range": true, "refId": "total", "useBackend": false }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "$available / $total", "hide": false, "refId": "perc", "type": "math" } ], "title": "Memory", "type": "timeseries" } ], "preload": false, "refresh": "10s", "schemaVersion": 42, "tags": [], "templating": { "list": [ { "current": { "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "definition": "label_values(systemd_unit_state{name=~\".*backup.*\", name=~\".*.service\", name!~\".*restore.*\", name!~\".*pre.service\"},name)", "includeAll": true, "label": "Job", "multi": true, "name": "job", "options": [], "query": { "qryType": 1, "query": "label_values(systemd_unit_state{name=~\".*backup.*\", name=~\".*.service\", name!~\".*restore.*\", name!~\".*pre.service\"},name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "/(?.*).service/", "sort": 1, "type": "query" }, { "current": { "text": [ "/srv/backup" ], "value": [ "/srv/backup" ] }, "definition": "label_values(node_filesystem_avail_bytes,mountpoint)", "includeAll": true, "multi": true, "name": "mountpoints", "options": [], "query": { "qryType": 1, "query": "label_values(node_filesystem_avail_bytes,mountpoint)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Backups", "uid": "f05500d0-15ed-4719-b68d-fb898ca13cc8", "version": 62 }