[
  {
    "uid": "f5246fa3-163f-4eae-9e1d-5b0fe2af0509",
    "title": "5XX Requests Error Budgets Under 99%",
    "condition": "threshold",
    "data": [
      {
        "refId": "A",
        "queryType": "range",
        "relativeTimeRange": {
          "from": 21600,
          "to": 0
        },
        "datasourceUid": "cd6cc53e-840c-484d-85f7-96fede324006",
        "model": {
          "datasource": {
            "type": "loki",
            "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
          },
          "editorMode": "code",
          "expr": "(sum by(server_name) (count_over_time({unit=\"nginx.service\"} | pattern \"<_> <_> <line>\" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | status =~ \"[1234]..\" | server_name =~ \".*\" [1h])) / sum by(server_name) (count_over_time({unit=\"nginx.service\"} | pattern \"<_> <_> <line>\" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | server_name =~ \".*\" [1h])))",
          "intervalMs": 1000,
          "legendFormat": "{{server_name}}",
          "maxDataPoints": 43200,
          "queryType": "range",
          "refId": "A"
        }
      },
      {
        "refId": "last",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [],
                "type": "gt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "B"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "A",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "reducer": "last",
          "refId": "last",
          "type": "reduce"
        }
      },
      {
        "refId": "threshold",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [
                  0.99
                ],
                "type": "lt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "C"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "last",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "refId": "threshold",
          "type": "threshold"
        }
      }
    ],
    "dasboardUid": "d66242cf-71e8-417c-8ef7-51b0741545df",
    "panelId": 9,
    "noDataState": "OK",
    "execErrState": "Error",
    "for": "20m",
    "annotations": {
      "__dashboardUid__": "d66242cf-71e8-417c-8ef7-51b0741545df",
      "__panelId__": "9",
      "description": "",
      "runbook_url": "",
      "summary": "The error budget for a service for the last 1 hour is under 99%"
    },
    "labels": {
      "role": "sysadmin"
    },
    "isPaused": false
  },
  {
    "uid": "ee817l3a88s1sd",
    "title": "Certificate Did Not Renew",
    "condition": "C",
    "data": [
      {
        "refId": "A",
        "relativeTimeRange": {
          "from": 1800,
          "to": 0
        },
        "datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09",
        "model": {
          "adhocFilters": [],
          "datasource": {
            "type": "prometheus",
            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
          },
          "editorMode": "code",
          "expr": "min by(subject) (ssl_certificate_expiry_seconds)",
          "interval": "",
          "intervalMs": 15000,
          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
          "maxDataPoints": 43200,
          "range": true,
          "refId": "A"
        }
      },
      {
        "refId": "B",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [],
                "type": "gt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "B"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "A",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "reducer": "last",
          "refId": "B",
          "type": "reduce"
        }
      },
      {
        "refId": "C",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [
                  604800
                ],
                "type": "lt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "C"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "B",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "refId": "C",
          "type": "threshold"
        }
      }
    ],
    "dashboardUid": "ae818js0bvw8wb",
    "panelId": 3,
    "noDataState": "NoData",
    "execErrState": "Error",
    "for": "20m",
    "annotations": {
      "__dashboardUid__": "ae818js0bvw8wb",
      "__panelId__": "3",
      "description": "The expiry date of the certificate is 1 week from now.",
      "summary": "Certificate did not renew on time."
    },
    "labels": {
      "role": "sysadmin"
    },
    "isPaused": false
  },
  {
    "uid": "df4doj5pomhvkf",
    "title": "Late Backups",
    "condition": "C",
    "data": [
      {
        "refId": "A",
        "relativeTimeRange": {
          "from": 10800,
          "to": 0
        },
        "datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09",
        "model": {
          "adhocFilters": [],
          "datasource": {
            "type": "prometheus",
            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "(\n  # Timer triggered at least once in the last 24h\n  label_replace((\n    time()\n      -\n    systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n  ) < 24*60*60, \"name\", \"$1.service\", \"name\", \"(.*).timer\")\n  AND on(name)\n  # At least one failure in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"failed\"}[24h]) > 0\n  )\n  AND on(name)\n  # No successes in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"inactive\"}[24h]) == 0\n  )\n)",
          "instant": false,
          "interval": "",
          "intervalMs": 15000,
          "legendFormat": "{{name}}",
          "maxDataPoints": 43200,
          "range": true,
          "refId": "A"
        }
      },
      {
        "refId": "B",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [],
                "type": "gt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "B"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "A",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "reducer": "last",
          "refId": "B",
          "type": "reduce"
        }
      },
      {
        "refId": "C",
        "relativeTimeRange": {
          "from": 0,
          "to": 0
        },
        "datasourceUid": "__expr__",
        "model": {
          "conditions": [
            {
              "evaluator": {
                "params": [
                  0
                ],
                "type": "gt"
              },
              "operator": {
                "type": "and"
              },
              "query": {
                "params": [
                  "C"
                ]
              },
              "reducer": {
                "params": [],
                "type": "last"
              },
              "type": "query"
            }
          ],
          "datasource": {
            "type": "__expr__",
            "uid": "__expr__"
          },
          "expression": "B",
          "intervalMs": 1000,
          "maxDataPoints": 43200,
          "refId": "C",
          "type": "threshold"
        }
      }
    ],
    "dashboardUid": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
    "panelId": 15,
    "noDataState": "OK",
    "execErrState": "Error",
    "annotations": {
      "__dashboardUid__": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
      "__panelId__": "15",
      "summary": "A backup did not run in the last 24 hours."
    },
    "labels": {
      "role": "sysadmin"
    },
    "isPaused": false
  }
]