diff --git a/kubernetes/apps/default/homelab/ks.yaml b/kubernetes/apps/default/homelab/ks.yaml index fa7a3e48f..4d7abe0e6 100644 --- a/kubernetes/apps/default/homelab/ks.yaml +++ b/kubernetes/apps/default/homelab/ks.yaml @@ -46,3 +46,27 @@ spec: postBuild: substitute: APP: *app +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app homelab-nas + namespace: flux-system +spec: + targetNamespace: default + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/default/homelab/nas + prune: true + sourceRef: + kind: GitRepository + name: home-ops-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/apps/observability/kube-prometheus-stack/rules/kustomization.yaml b/kubernetes/apps/default/homelab/nas/kustomization.yaml similarity index 85% rename from kubernetes/apps/observability/kube-prometheus-stack/rules/kustomization.yaml rename to kubernetes/apps/default/homelab/nas/kustomization.yaml index ded29f67c..800d5b6ee 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/rules/kustomization.yaml +++ b/kubernetes/apps/default/homelab/nas/kustomization.yaml @@ -2,5 +2,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: default resources: - - ./zfs.yaml + - ./rules.yaml diff --git a/kubernetes/apps/observability/grafana/dashboards/homelab-temperatures.json b/kubernetes/apps/default/homelab/nas/resticprofile-dashboard.json similarity index 53% rename from kubernetes/apps/observability/grafana/dashboards/homelab-temperatures.json rename to kubernetes/apps/default/homelab/nas/resticprofile-dashboard.json index 9c35ed040..f2cc5c445 100644 --- a/kubernetes/apps/observability/grafana/dashboards/homelab-temperatures.json +++ b/kubernetes/apps/default/homelab/nas/resticprofile-dashboard.json @@ -4,19 +4,13 @@ { "builtIn": 1, "datasource": { - "t²ype": "datasource", - "uid": "grafana" + "type": "grafana", + "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, "type": "dashboard" } ] @@ -24,14 +18,101 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, "links": [], - "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 2, + "text": "Fail" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Warning" + }, + "2": { + "color": "green", + "index": 0, + "text": "Success" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "resticprofile_backup_status", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{profile}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Backup status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -39,11 +120,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -52,14 +135,15 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -68,7 +152,6 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -83,94 +166,68 @@ } ] }, - "unit": "celsius" + "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Value" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, + "h": 8, + "w": 12, + "x": 12, "y": 0 }, - "id": 5, + "id": 1, "options": { "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, - "pluginVersion": "8.0.5", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(node_cpu_temperature_celsius{app=\"opnsense\"})", - "interval": "", - "legendFormat": "Temperature", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "resticprofile_backup_duration_seconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{profile}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "OPNsense - CPU", + "title": "Backup duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, - "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -179,103 +236,95 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "line+area" + "mode": "off" } }, - "decimals": 0, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "transparent", + "color": "green", "value": null }, - { - "color": "orange", - "value": 80 - }, { "color": "red", - "value": 90 + "value": 80 } ] }, - "unit": "celsius" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 9 + "y": 8 }, "id": 2, "options": { "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, - "pluginVersion": "8.0.5", + "pluginVersion": "11.4.0", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "avg (node_hwmon_temp_celsius{chip=\"platform_coretemp_0\"}) by (chip,kubernetes_node) * ignoring(chip_name) group_left(chip_name) avg (node_hwmon_chip_names) by (chip,chip_name,kubernetes_node)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{chip_name}}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "resticprofile_backup_files_processed", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{profile}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "k3s Nodes - CPU", - "transformations": [], + "title": "Files processes", "type": "timeseries" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -284,112 +333,92 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "line+area" + "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "transparent", + "color": "green", "value": null }, { "color": "red", - "value": 40 + "value": 80 } ] }, - "unit": "celsius" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 18 + "h": 8, + "w": 12, + "x": 12, + "y": 8 }, - "id": 6, + "id": 3, "options": { "legend": { - "calcs": [ - "mean", - "max", - "min" - ], + "calcs": [], "displayMode": "list", - "placement": "right", - "showLegend": false + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, - "pluginVersion": "8.0.5", + "pluginVersion": "11.4.0", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(hass_sensor_temperature_celsius{entity=~\"sensor.temperature_servers_temperature\"})", - "interval": "", - "legendFormat": "datacenter", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "resticprofile_backup_processed_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{profile}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Datacenter", + "title": "Processed bytes", "type": "timeseries" } ], - "schemaVersion": 37, - "style": "dark", - "tags": [ - "homelab" - ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], "templating": { "list": [] }, "time": { - "from": "now-24h", + "from": "now-12h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Homelab / Temperatures", - "uid": "aEY0BVGnz", - "version": 1, + "timepicker": {}, + "timezone": "browser", + "title": "New dashboard", + "version": 0, "weekStart": "" } diff --git a/kubernetes/apps/default/homelab/nas/rules.yaml b/kubernetes/apps/default/homelab/nas/rules.yaml new file mode 100644 index 000000000..53e8ab6e6 --- /dev/null +++ b/kubernetes/apps/default/homelab/nas/rules.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: resticprofile-backup-status + namespace: monitoring +spec: + groups: + - name: resticprofile.rules + rules: + - alert: ResticProfileBackupWarning + expr: resticprofile_backup_status == 1 + annotations: + description: "The {{ $labels.profile }} profile status on hostname {{ $labels.group }} is Warning" + labels: + severity: warning + + - alert: ResticProfileBackupFailure + expr: resticprofile_backup_status == 0 + annotations: + description: "The {{ $labels.profile }} profile status on hostname {{ $labels.group }} is Fail" + labels: + severity: critical diff --git a/kubernetes/apps/observability/grafana/app/helmrelease.yaml b/kubernetes/apps/observability/grafana/app/helmrelease.yaml index 846538314..ea7267fc5 100644 --- a/kubernetes/apps/observability/grafana/app/helmrelease.yaml +++ b/kubernetes/apps/observability/grafana/app/helmrelease.yaml @@ -164,11 +164,8 @@ spec: implementation: prometheus dashboards: default: - home-assistant: - url: https://raw.githubusercontent.com/auricom/home-ops/main/kubernetes/apps/observability/grafana/dashboards/home-assistant.json - datasource: Prometheus - homelab-temperatures: - url: https://raw.githubusercontent.com/auricom/home-ops/main/kubernetes/apps/observability/grafana/dashboards/homelab-temperatures.json + resticprofile: + url: https://raw.githubusercontent.com/auricom/home-ops/main/kubernetes/apps/default/homelab/nas/resticprofile-dashboard.json datasource: Prometheus external-dns: # renovate: depName="External-dns" @@ -204,17 +201,11 @@ spec: node-feature-discovery: url: https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/master/examples/grafana-dashboard.json datasource: Prometheus - resticprofile: - url: https://raw.githubusercontent.com/creativeprojects/resticprofile/047b81702b0a583a61facd72e704ab4f7bd495eb/contrib/grafana/dashboard.json - datasource: Prometheus zfs: # renovate: depName="ZFS" gnetId: 7845 revision: 4 datasource: Prometheus - zrepl: - url: https://raw.githubusercontent.com/zrepl/zrepl/master/dist/grafana/grafana-prometheus-zrepl.json - datasource: Prometheus ceph: ceph-cluster: # renovate: depName="Ceph Cluster" diff --git a/kubernetes/apps/observability/grafana/dashboards/home-assistant.json b/kubernetes/apps/observability/grafana/dashboards/home-assistant.json deleted file mode 100644 index 936e2ecd9..000000000 --- a/kubernetes/apps/observability/grafana/dashboards/home-assistant.json +++ /dev/null @@ -1,437 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 40 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(hass_sensor_temperature_celsius{entity=\"sensor.temperature_living_room_temperature\"})", - "instant": false, - "interval": "", - "legendFormat": "Living Room", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(hass_sensor_temperature_celsius{entity=~\"sensor.temperature_bedroom_temperature\"})", - "hide": false, - "interval": "", - "legendFormat": "Bedroom 1", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(hass_sensor_temperature_celsius{entity=~\"sensor.temperature_office_temperature\"})", - "hide": false, - "interval": "", - "legendFormat": "Office", - "range": true, - "refId": "C" - } - ], - "title": "Home", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(hass_sensor_power_w{entity=\"sensor.servers_plug_switch_electricity_power\"})", - "interval": "", - "legendFormat": "Servers", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(hass_sensor_power_w{entity=\"sensor.bureau_plug_switch_electricity_power\"})", - "hide": false, - "interval": "", - "legendFormat": "claude-fixe", - "range": true, - "refId": "B" - } - ], - "title": "Computers", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(hass_sensor_power_w{entity=~\"sensor.power_plug_avr_power\"})", - "interval": "", - "legendFormat": "AVR", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "hass_sensor_power_w{entity=~\"sensor.power_plug_projector_power\"}", - "hide": false, - "interval": "", - "legendFormat": "Projector", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "hass_sensor_power_w{entity=~\"sensor.power_plug_subwoofer_power\"}", - "hide": false, - "interval": "", - "legendFormat": "Subwoofer", - "range": true, - "refId": "C" - } - ], - "title": "Home Cinema", - "type": "timeseries" - } - ], - "schemaVersion": 37, - "style": "dark", - "tags": [ - "home-assistant" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Home-Assistant", - "uid": "sn-bOoWMk", - "version": 1, - "weekStart": "" -} diff --git a/kubernetes/apps/observability/grafana/dashboards/truenas.json b/kubernetes/apps/observability/grafana/dashboards/truenas.json deleted file mode 100644 index f206511b0..000000000 --- a/kubernetes/apps/observability/grafana/dashboards/truenas.json +++ /dev/null @@ -1,942 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.1.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "limit": 100, - "name": "Annotations & Alerts", - "showIn": 0, - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1630317513315, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "panels": [], - "repeat": null, - "title": "Summary", - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "rgb(255, 255, 255)", - "value": 600 - } - ] - }, - "unit": "dtdurations" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 0, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "value" - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "exemplar": true, - "expr": "system_uptime{host=\"$host\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "semi-dark-yellow", - "value": 80 - }, - { - "color": "semi-dark-red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 2, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "exemplar": true, - "expr": "sum(100-cpu_usage_idle{host=\"$host\", cpu=\"cpu-total\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU used", - "type": "gauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "semi-dark-yellow", - "value": 95 - }, - { - "color": "semi-dark-red", - "value": 98 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 6, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "exemplar": true, - "expr": "mem_used_percent{host=\"$host\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory used", - "type": "gauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "semi-dark-yellow", - "value": 70 - }, - { - "color": "semi-dark-red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 10, - "y": 1 - }, - "id": 8, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": { - "titleSize": 1 - } - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "exemplar": true, - "expr": "zfs_pool_capacity{host=\"$host\", pool=~\"storage|vol1\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pool capacity", - "type": "gauge" - }, - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 12, - "panels": [], - "title": "System", - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": true, - "expr": "sum(100-cpu_usage_idle{host=\"$host\", cpu=\"cpu-total\"})", - "interval": "", - "legendFormat": "CPU usage", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": true, - "expr": "rate(net_bytes_recv{host=\"$host\", interface=~\"em0|igb0\"}[5m])", - "interval": "", - "legendFormat": "received by {{interface}}", - "refId": "A" - }, - { - "exemplar": true, - "expr": "rate(net_bytes_sent{host=\"$host\", interface=~\"em0|igb0\"}[5m])", - "hide": false, - "interval": "", - "legendFormat": "sent by {{interface}}", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Usage", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": true, - "expr": "sum(temp_temp{host=\"$host\"}) / 4", - "interval": "", - "legendFormat": "Temperature", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU temperature", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": false, - "expr": "sum(disktemp_temperature{host=\"$host\"}) by (disk)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{disk}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Disks temperatures", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 17, - "panels": [], - "title": "Block devices", - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(diskio_read_bytes{host=\"$host\",name=~\"ada[0-9]|nvd[0-9]\"}[5m])) by (name)", - "interval": "", - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Reads", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.5", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(diskio_write_bytes{host=\"$host\",name=~\"ada[0-9]|nvd[0-9]\"}[5m])) by (name)", - "interval": "", - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Writes", - "type": "timeseries" - } - ], - "schemaVersion": 30, - "style": "dark", - "tags": [ - "truenas", - "telegraf" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(system_uptime, host)", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "host", - "options": [], - "query": { - "query": "label_values(system_uptime, host)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/(.*)/", - "skipUrlSync": false, - "sort": 2, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "TrueNAS", - "uid": "l65MB4M7z", - "version": 1 -} diff --git a/kubernetes/apps/observability/kube-prometheus-stack/ks.yaml b/kubernetes/apps/observability/kube-prometheus-stack/ks.yaml index c77549ea0..4d3754512 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/ks.yaml +++ b/kubernetes/apps/observability/kube-prometheus-stack/ks.yaml @@ -47,26 +47,3 @@ spec: APP: *app # renovate: datasource=docker depName=quay.io/thanos/thanos THANOS_VERSION: v0.35.0 ---- -# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: &app kube-prometheus-rules - namespace: flux-system -spec: - targetNamespace: observability - commonMetadata: - labels: - app.kubernetes.io/name: *app - dependsOn: - - name: kube-prometheus-stack - path: ./kubernetes/apps/observability/kube-prometheus-stack/rules - prune: true - sourceRef: - kind: GitRepository - name: home-ops-kubernetes - wait: false - interval: 30m - retryInterval: 1m - timeout: 5m diff --git a/kubernetes/apps/observability/kube-prometheus-stack/rules/zfs.yaml b/kubernetes/apps/observability/kube-prometheus-stack/rules/zfs.yaml deleted file mode 100644 index e3b5aea66..000000000 --- a/kubernetes/apps/observability/kube-prometheus-stack/rules/zfs.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: zrepl-replication-errors -spec: - groups: - - name: zrepl.rules - rules: - - alert: ZreplReplicationFilesystemErrors - expr: zrepl_replication_filesystem_errors > 0 - for: 15m # Alert if the condition persists for 15 minutes - labels: - severity: warning - annotations: - summary: "Zrepl replication filesystem errors detected" - description: "Zrepl job {{ $labels.zrepl_job }} has encountered filesystem errors during replication."