feat: benji alerting

cluster/apps/rook-system/benji/kustomization.yaml

@@ -6,4 +6,5 @@ resources:
   - volume.yaml
   - helm-release.yaml
   - podmonitor.yaml
+  - prometheus-rule.yaml
   - pgbackups.yaml

cluster/apps/rook-system/benji/prometheus-rule.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: benji
+  namespace: rook-ceph
+spec:
+  groups:
+    - name: Benji
+      rules:
+        - alert: BackupFailed
+          annotations:
+            summary: >-
+              {{ $labels.volume }} backup has failed.
+          expr: >-
+            benji_backup_status_succeeded != 1
+          for: 30m
+          labels:
+            severity: page