♻️ kube-tools

ansible/inventory/group_vars/worker/k3s.yml

@@ -14,3 +14,5 @@ k3s_agent:
     # Allow pods to be rescheduled quicker in the case of a node failure
     # https://github.com/k3s-io/k3s/issues/1264
     - "node-status-update-frequency=4s"
+  node-label:
+    - node-role.kubernetes.io/worker=true

cluster/apps/authentication/authelia/helm-release.yaml

@@ -50,8 +50,7 @@ spec:
         enabled: true
         ingressClassName: "nginx"
         annotations:
-          external-dns.alpha.kubernetes.io/target: "services.${SECRET_DOMAIN}."
-          external-dns/is-public: "true"
+          external-dns.home.arpa/enabled: "true"
           nginx.ingress.kubernetes.io/configuration-snippet: |
             add_header Cache-Control "no-store";
             add_header Pragma "no-cache";

cluster/apps/downloaders/sabnzbd/helm-release.yaml

@@ -48,8 +48,7 @@ spec:
         enabled: true
         ingressClassName: "nginx"
         annotations:
-          nginx.ingress.kubernetes.io/auth-url: "http://authelia.default.svc.cluster.local/api/verify"
-          nginx.ingress.kubernetes.io/auth-signin: "https://auth.${SECRET_CLUSTER_DOMAIN}"
+          auth.home.arpa/enabled: "true"
           nginx.ingress.kubernetes.io/configuration-snippet: |
             proxy_set_header Accept-Encoding "";
             sub_filter '</head>' '<link rel="stylesheet" type="text/css" href="https://theme-park.${SECRET_CLUSTER_DOMAIN}/css/base/sabnzbd/nord.css"></head>';

cluster/apps/kube-tools/coredns-nodecache/configmap.yaml

@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: coredns-nodecache-primary
-  namespace: kube-system
+  namespace: default
 data:
   Corefile: |
     cluster.local:53 {
@@ -65,7 +65,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: coredns-nodecache-secondary
-  namespace: kube-system
+  namespace: default
 data:
   Corefile: |
     cluster.local:53 {

cluster/apps/kube-tools/descheduler/helm-release.yaml

@@ -0,0 +1,86 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app descheduler
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: *app
+      version: 0.24.1
+      sourceRef:
+        kind: HelmRepository
+        name: descheduler-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    kind: Deployment
+    replicas: 1
+    leaderElection:
+      enabled: true
+      leaseDuration: 15s
+      renewDeadline: 10s
+      retryPeriod: 2s
+      resourceLock: "leases"
+      resourceName: "descheduler"
+      resourceNamescape: "kube-system"
+    deschedulerPolicy:
+      strategies:
+        RemoveDuplicates:
+          enabled: true
+        RemovePodsViolatingNodeTaints:
+          enabled: true
+        RemovePodsViolatingNodeAffinity:
+          enabled: true
+          params:
+            nodeAffinityType:
+              - requiredDuringSchedulingIgnoredDuringExecution
+        RemovePodsViolatingTopologySpreadConstraint:
+          enabled: true
+          params:
+            includeSoftConstraints: true
+        RemovePodsViolatingInterPodAntiAffinity:
+          enabled: true
+          params:
+            nodeFit: true
+        LowNodeUtilization:
+          enabled: false
+        RemoveFailedPods:
+          enabled: true
+          params:
+            failedPods:
+              includingInitContainers: true
+              excludeOwnerKinds:
+                - "Job"
+              minPodLifetimeSeconds: 3600
+        RemovePodsHavingTooManyRestarts:
+          enabled: true
+          params:
+            podsHavingTooManyRestarts:
+              podRestartThreshold: 100
+              includingInitContainers: true
+    service:
+      enabled: true
+    serviceMonitor:
+      enabled: true
+    podAnnotations:
+      configmap.reloader.stakater.com/reload: *app
+    affinity:
+      podAntiAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                  - key: app.kubernetes.io/name
+                    operator: In
+                    values: [*app]
+              topologyKey: kubernetes.io/hostname

cluster/apps/kube-tools/descheduler/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/kube-tools/goldilocks/helm-release.yaml

@@ -0,0 +1,50 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: goldilocks
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: goldilocks
+      version: 6.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: fairwinds-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    controller:
+      resources:
+        requests:
+          cpu: 10m
+          memory: 64M
+        limits:
+          memory: 250M
+    dashboard:
+      replicaCount: 1
+      ingress:
+        enabled: true
+        ingressClassName: "nginx"
+        hosts:
+          - host: &host "goldilocks.${SECRET_CLUSTER_DOMAIN}"
+            paths:
+              - path: /
+                type: Prefix
+        tls:
+          - hosts:
+              - *host
+      resources:
+        requests:
+          cpu: 10m
+          memory: 50Mi
+        limits:
+          memory: 150Mi

cluster/apps/kube-tools/goldilocks/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/kube-tools/intel-gpu-exporter/helm-release.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app intel-gpu-exporter
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: app-template
+      version: 0.1.1
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: intel-gpu-plugin
+      namespace: default
+  values:
+    controller:
+      type: daemonset
+    image:
+      repository: ghcr.io/onedr0p/intel-gpu-exporter
+      tag: rolling@sha256:1c84020b442e0f95e2e6a46281d3bfc6199902d3d91b10515fc000e43c9a7421
+    service:
+      main:
+        ports:
+          http:
+            port: 8080
+    securityContext:
+      privileged: true
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: feature.node.kubernetes.io/custom-intel-gpu
+                  operator: In
+                  values:
+                    - "true"
+    resources:
+      requests:
+        gpu.intel.com/i915: 1
+        cpu: 100m
+        memory: 100Mi
+      limits:
+        gpu.intel.com/i915: 1
+        memory: 500Mi

cluster/apps/kube-tools/intel-gpu-exporter/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml
+  - service-monitor.yaml

cluster/apps/kube-tools/intel-gpu-exporter/service-monitor.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: &app intel-gpu-exporter
+  namespace: default
+  labels: &labels
+    app.kubernetes.io/instance: *app
+    app.kubernetes.io/name: *app
+spec:
+  selector:
+    matchLabels:
+      <<: *labels
+  endpoints:
+    - port: http
+      interval: 1m
+      scrapeTimeout: 10s
+      path: /metrics
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node

cluster/apps/kube-tools/intel-gpu-plugin/helm-release.yaml

@@ -0,0 +1,75 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app intel-gpu-plugin
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: app-template
+      version: 0.1.1
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: node-feature-discovery
+      namespace: default
+  values:
+    controller:
+      type: daemonset
+      strategy: RollingUpdate
+    image:
+      repository: docker.io/intel/intel-gpu-plugin
+      tag: 0.24.0
+      pullPolicy: IfNotPresent
+    args:
+      - -shared-dev-num
+      - "4"
+    service:
+      main:
+        enabled: false
+    # TODO(intel-gpu-plugin): Write probes to check for something to tell if it's working
+    probes:
+      liveness:
+        enabled: false
+      readiness:
+        enabled: false
+      startup:
+        enabled: false
+    persistence:
+      devfs:
+        enabled: true
+        type: hostPath
+        hostPath: /dev/dri
+        hostPathType: Directory
+        readOnly: true
+      sysfs:
+        enabled: true
+        type: hostPath
+        hostPath: /sys/class/drm
+        hostPathType: Directory
+        readOnly: true
+      kubeletsockets:
+        enabled: true
+        type: hostPath
+        hostPathType: Directory
+        hostPath: /var/lib/kubelet/device-plugins
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: feature.node.kubernetes.io/custom-intel-gpu
+                  operator: In
+                  values:
+                    - "true"

cluster/apps/kube-tools/intel-gpu-plugin/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/kube-tools/kured/helm-release.yaml

@@ -8,7 +8,6 @@ spec:
   interval: 5m
   chart:
     spec:
-      # renovate: registryUrl=https://weaveworks.github.io/kured
       chart: kured
       version: 3.0.1
       sourceRef:

cluster/apps/kube-tools/kustomization.yaml

@@ -1,10 +1,16 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
-  - coredns-nodecache
+  #- coredens-nodecache
   - descheduler
+  - goldilocks
+  - intel-gpu-exporter
   - intel-gpu-plugin
   - kured
+  - kyverno
+  - metrics-server
   - node-feature-discovery
   - reloader
+  - system-upgrade
+  - vpa

cluster/apps/kube-tools/kyverno/helm-release.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: kyverno
+  namespace: kyverno
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: kyverno
+      version: v2.5.3
+      sourceRef:
+        kind: HelmRepository
+        name: kyverno-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    installCRDs: false
+    replicaCount: 3
+    extraArgs:
+      - --autogenInternals=false
+      - --clientRateLimitQPS=30
+      - --clientRateLimitBurst=60
+    serviceMonitor:
+      enabled: true
+    topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+        labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: kyverno

cluster/apps/kube-tools/kyverno/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # renovate: registryUrl=https://kyverno.github.io/kyverno/ chart=kyverno
+  - github.com/kyverno/kyverno//config/crds?ref=kyverno-chart-v2.5.3
+  - helm-release.yaml
+  - rbac.yaml
+  - policies

cluster/apps/kube-tools/kyverno/policies/apply-ingress-auth-annotations.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: apply-ingress-auth-annotations
+  annotations:
+    policies.kyverno.io/title: Apply Ingress Auth Annotations
+    policies.kyverno.io/subject: Ingress
+    policies.kyverno.io/description: >-
+      This policy creates auth annotations on ingresses. When
+      the `auth.home.arpa/enabled` annotation is `true` it
+      applies the nginx auth annotations for use with Authelia.
+spec:
+  mutateExistingOnPolicyUpdate: true
+  generateExistingOnPolicyUpdate: true
+  rules:
+    - name: auth
+      match:
+        any:
+          - resources:
+              kinds: ["Ingress"]
+              annotations:
+                auth.home.arpa/enabled: "true"
+      mutate:
+        patchStrategicMerge:
+          metadata:
+            annotations:
+              +(nginx.ingress.kubernetes.io/auth-method): GET
+              +(nginx.ingress.kubernetes.io/auth-url): |-
+                http://authelia.default.svc.cluster.local/api/verify
+              +(nginx.ingress.kubernetes.io/auth-signin): |-
+                https://auth.${SECRET_CLUSTER_DOMAIN}?rm=$request_method
+              +(nginx.ingress.kubernetes.io/auth-response-headers): |-
+                Remote-User,Remote-Name,Remote-Groups,Remote-Email
+              +(nginx.ingress.kubernetes.io/auth-snippet): |
+                proxy_set_header X-Forwarded-Method $request_method;

cluster/apps/kube-tools/kyverno/policies/apply-ingress-external-dns-annotations.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: apply-ingress-external-dns-annotations
+  annotations:
+    policies.kyverno.io/title: Apply Ingress External-DNS Annotations
+    policies.kyverno.io/subject: Ingress
+    policies.kyverno.io/description: >-
+      This policy creates external-dns annotations on ingresses.
+      When the `external-dns.home.arpa/enabled` annotation is `true`
+      it applies the external-dns annotations for use with external
+      application access.
+spec:
+  mutateExistingOnPolicyUpdate: true
+  generateExistingOnPolicyUpdate: true
+  rules:
+    - name: external-dns
+      match:
+        any:
+          - resources:
+              kinds: ["Ingress"]
+              annotations:
+                external-dns.home.arpa/enabled: "true"
+      mutate:
+        patchStrategicMerge:
+          metadata:
+            annotations:
+              +(external-dns.alpha.kubernetes.io/target): |-
+                services.${SECRET_DOMAIN}.
+              +(external-dns/is-public): |-
+                true

cluster/apps/kube-tools/kyverno/policies/apply-ingress-whitelist-annotations.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: apply-ingress-whitelist-annotations
+  annotations:
+    policies.kyverno.io/title: Apply Ingress Whitelist Annotations
+    policies.kyverno.io/subject: Ingress
+    policies.kyverno.io/description: >-
+      This policy creates annotations on ingresses. When
+      the `external-dns.home.arpa/enabled` annotation is not
+      set it applies the nginx annotations for use with only
+      internal application access.
+spec:
+  mutateExistingOnPolicyUpdate: true
+  generateExistingOnPolicyUpdate: true
+  rules:
+    - name: whitelist
+      match:
+        any:
+          - resources:
+              kinds: ["Ingress"]
+      exclude:
+        any:
+          - resources:
+              annotations:
+                external-dns.home.arpa/enabled: "true"
+      mutate:
+        patchStrategicMerge:
+          metadata:
+            annotations:
+              +(nginx.ingress.kubernetes.io/whitelist-source-range): |-
+                10.0.0.0/8,172.16.0.0/12,192.168.0.0/16

cluster/apps/kube-tools/kyverno/policies/delete-cpu-limits.yaml

@@ -0,0 +1,51 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: delete-cpu-limits
+  annotations:
+    policies.kyverno.io/title: Delete CPU limits
+    policies.kyverno.io/subject: Pod
+    policies.kyverno.io/description: >-
+      This policy deletes CPU limits from all Pods.
+spec:
+  mutateExistingOnPolicyUpdate: true
+  generateExistingOnPolicyUpdate: true
+  rules:
+    - name: delete-cpu-limits
+      match:
+        any:
+          - resources:
+              kinds: ["Pod"]
+      exclude:
+        any:
+          - resources:
+              namespaces:
+                - calico-system
+                - tigera-operator
+          - resources:
+              kinds: ["Pod"]
+              selector:
+                matchLabels:
+                  job-name: "*"
+          - resources:
+              kinds: ["Pod"]
+              selector:
+                matchLabels:
+                  statefulset.kubernetes.io/pod-name: "*"
+          - resources:
+              annotations:
+                kyverno.io/ignore: "true"
+      mutate:
+        patchStrategicMerge:
+          spec:
+            initContainers:
+              - (name): "*"
+                resources:
+                  limits:
+                    cpu: null
+            containers:
+              - (name): "*"
+                resources:
+                  limits:
+                    cpu: null

cluster/apps/kube-tools/kyverno/policies/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - apply-ingress-auth-annotations.yaml
+  - apply-ingress-external-dns-annotations.yaml
+  - apply-ingress-whitelist-annotations.yaml
+  - delete-cpu-limits.yaml
+  - snapshot-cronjob-controller.yaml
+  - sync-postgres-secrets.yaml

cluster/apps/kube-tools/kyverno/policies/snapshot-cronjob-controller.yaml

@@ -0,0 +1,137 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: snapshot-cronjob-controller
+  annotations:
+    policies.kyverno.io/title: Snapshot CronJob controller
+    policies.kyverno.io/subject: PersistentVolumeClaim
+    policies.kyverno.io/description: |
+      This policy creates a Kopia snapshot CronJob for labeled PersistentVolumeClaims
+
+      The following labels on PVCs with their respective labels are required for this to run:
+        - snapshot.home.arpa/enabled
+        - app.kubernetes.io/name
+        - app.kubernetes.io/instance
+
+      An optional label of "snapshot.home.arpa/ignoreAffinity" may be set on the PVC
+      if the pod is guaranteed to not run during the time of this jobs execution
+spec:
+  generateExistingOnPolicyUpdate: true
+  mutateExistingOnPolicyUpdate: true
+  rules:
+    - name: create-snapshot-cronjob
+      match:
+        any:
+          - resources:
+              kinds:
+                - PersistentVolumeClaim
+              selector:
+                matchLabels:
+                  snapshot.home.arpa/enabled: "true"
+                  app.kubernetes.io/name: "*"
+                  app.kubernetes.io/instance: "*"
+      context:
+        - name: appName
+          variable:
+            jmesPath: "request.object.metadata.labels.\"app.kubernetes.io/name\""
+        - name: claimName
+          variable:
+            jmesPath: "request.object.metadata.name"
+        - name: namespace
+          variable:
+            jmesPath: "request.object.metadata.namespace"
+        - name: nodeAffinity
+          variable:
+            value:
+              ignored: "{{ (request.object.metadata.labels.\"snapshot.home.arpa/ignoreAffinity\" || 'false') == 'false' }}"
+              labels:
+                - key: app.kubernetes.io/name
+                  operator: "In"
+                  values:
+                    - "{{ request.object.metadata.labels.\"app.kubernetes.io/name\" }}"
+                - key: app.kubernetes.io/instance
+                  operator: "In"
+                  values:
+                    - "{{ request.object.metadata.labels.\"app.kubernetes.io/instance\" }}"
+      generate:
+        synchronize: true
+        apiVersion: batch/v1
+        kind: CronJob
+        name: "{{ appName }}-{{ claimName }}-snapshot"
+        namespace: "{{ request.object.metadata.namespace }}"
+        data:
+          metadata:
+            labels:
+              app.kubernetes.io/name: "{{ request.object.metadata.labels.\"app.kubernetes.io/name\" }}"
+              app.kubernetes.io/instance: "{{ request.object.metadata.labels.\"app.kubernetes.io/instance\" }}"
+            ownerReferences:
+              - apiVersion: "{{ request.object.apiVersion }}"
+                kind: "{{ request.object.kind }}"
+                name: "{{ request.object.metadata.name }}"
+                uid: "{{ request.object.metadata.uid }}"
+          spec:
+            schedule: "0 3 * * *"
+            suspend: false
+            concurrencyPolicy: Forbid
+            successfulJobsHistoryLimit: 1
+            failedJobsHistoryLimit: 2
+            jobTemplate:
+              spec:
+                # Keep at least one job in completed state in accordance to the schedule
+                ttlSecondsAfterFinished: 86400
+                template:
+                  spec:
+                    automountServiceAccountToken: false
+                    restartPolicy: OnFailure
+                    # Stagger jobs to run randomly within X seconds to avoid bringing down all apps at once
+                    initContainers:
+                      - name: wait
+                        image: ghcr.io/onedr0p/kopia:0.11.3@sha256:72406602c99357951cb7284abbf88699081d60f6cffd22baddd8a6a2afe919f5
+                        command: ["/scripts/sleep.sh"]
+                        args: ["1", "900"]
+                    containers:
+                      - name: snapshot
+                        image: ghcr.io/onedr0p/kopia:0.11.3@sha256:72406602c99357951cb7284abbf88699081d60f6cffd22baddd8a6a2afe919f5
+                        env:
+                          - name: KOPIA_CACHE_DIRECTORY
+                            value: /snapshots/{{ namespace }}/{{ appName }}/{{ claimName }}/cache
+                          - name: KOPIA_LOG_DIR
+                            value: /snapshots/{{ namespace }}/{{ appName }}/{{ claimName }}/logs
+                          - name: KOPIA_PASSWORD
+                            value: "none"
+                        command:
+                          - /bin/bash
+                          - -c
+                          - |-
+                            printf "\e[1;32m%-6s\e[m\n" "[01/10] Create repo ..."              && [[ ! -f /snapshots/kopia.repository.f ]] && kopia repository create filesystem --path=/snapshots
+                            printf "\e[1;32m%-6s\e[m\n" "[02/10] Connect to repo ..."          && kopia repo connect filesystem --path=/snapshots --override-hostname=cluster --override-username=root
+                            printf "\e[1;32m%-6s\e[m\n" "[03/10] Set policies ..."             && kopia policy set /data/{{ namespace }}/{{ appName }}/{{ claimName }} --compression=zstd --keep-latest 14 --keep-hourly 0 --keep-daily 7 --keep-weekly 2 --keep-monthly 0 --keep-annual 0
+                            printf "\e[1;32m%-6s\e[m\n" "[04/10] Freeze {{ claimName }} ..."   && fsfreeze -f /data/{{ namespace }}/{{ appName }}/{{ claimName }}
+                            printf "\e[1;32m%-6s\e[m\n" "[05/10] Snapshot {{ claimName }} ..." && kopia snap create /data/{{ namespace }}/{{ appName }}/{{ claimName }}
+                            printf "\e[1;32m%-6s\e[m\n" "[06/10] Unfreeze {{ claimName }} ..." && fsfreeze -u /data/{{ namespace }}/{{ appName }}/{{ claimName }}
+                            printf "\e[1;32m%-6s\e[m\n" "[07/10] List snapshots ..."           && kopia snap list /data/{{ namespace }}/{{ appName }}/{{ claimName }}
+                            printf "\e[1;32m%-6s\e[m\n" "[08/10] Show stats ..."               && kopia content stats
+                            printf "\e[1;32m%-6s\e[m\n" "[09/10] Show maintenance info ..."    && kopia maintenance info
+                            printf "\e[1;32m%-6s\e[m\n" "[10/10] Disconnect from repo ..."     && kopia repo disconnect
+                        volumeMounts:
+                          - name: data
+                            mountPath: "/data/{{ namespace }}/{{ appName }}/{{ claimName }}"
+                          - name: snapshots
+                            mountPath: /snapshots
+                        securityContext:
+                          privileged: true
+                    volumes:
+                      - name: data
+                        persistentVolumeClaim:
+                          claimName: "{{ claimName }}"
+                      - name: snapshots
+                        nfs:
+                          server: "expanse.${SECRET_PRIVATE_DOMAIN}"
+                          path: /eros/Apps/Kopia
+                    affinity:
+                      podAffinity:
+                        requiredDuringSchedulingIgnoredDuringExecution:
+                          - topologyKey: kubernetes.io/hostname
+                            labelSelector:
+                              matchExpressions: "{{ nodeAffinity.ignored && [] || nodeAffinity.labels }}"

cluster/apps/kube-tools/kyverno/policies/sync-postgres-secrets.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: sync-postgres-secrets
+  annotations:
+    policies.kyverno.io/title: Sync Postgres Secrets
+    policies.kyverno.io/subject: Secret
+    policies.kyverno.io/description: >-
+      This policy will copy a secret called `postgres-superuser` which
+      exists in the `database` namespace to new namespaces when they are
+      created. It will also push updates to the copied Secrets should the
+      source secret be changed.
+spec:
+  mutateExistingOnPolicyUpdate: true
+  generateExistingOnPolicyUpdate: true
+  rules:
+    - name: sync-postgres-superuser-secret
+      match:
+        resources:
+          kinds: ["Namespace"]
+      exclude:
+        resources:
+          namespaces: ["default"]
+      generate:
+        apiVersion: v1
+        kind: Secret
+        name: postgres-superuser
+        namespace: "{{request.object.metadata.name}}"
+        synchronize: true
+        clone:
+          namespace: default
+          name: postgres-superuser

cluster/apps/kube-tools/kyverno/rbac.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kyverno:admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: admin
+subjects:
+  - kind: ServiceAccount
+    name: kyverno
+    namespace: kyverno

cluster/apps/kube-tools/metrics-server/helm-release.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: metrics-server
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: metrics-server
+      version: 3.8.2
+      sourceRef:
+        kind: HelmRepository
+        name: metrics-server-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    args:
+      - --kubelet-insecure-tls
+      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+      - --kubelet-use-node-status-port
+      - --metric-resolution=15s
+    metrics:
+      enabled: true
+    serviceMonitor:
+      enabled: true

cluster/apps/kube-tools/metrics-server/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/kube-tools/node-feature-discovery/helm-release.yaml

@@ -3,29 +3,32 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
   name: node-feature-discovery
-  namespace: kube-system
+  namespace: default
 spec:
-  interval: 5m
+  interval: 15m
   chart:
     spec:
-      # renovate: registryUrl=https://kubernetes-sigs.github.io/node-feature-discovery/charts
       chart: node-feature-discovery
       version: 0.11.2
       sourceRef:
         kind: HelmRepository
         name: node-feature-discovery-charts
         namespace: flux-system
-      interval: 5m
   install:
     createNamespace: true
     remediation:
       retries: 5
+  upgrade:
+    remediation:
+      retries: 5
   values:
     nodeFeatureRule:
       createCRD: false
     worker:
       annotations:
         configmap.reloader.stakater.com/reload: node-feature-discovery-worker-conf
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
       config:
         core:
           sources:
@@ -42,9 +45,9 @@ spec:
               - "fe"
               - "ff"
             deviceLabelFields:
-              - "class"
-              - "vendor"
-              - "device"
+              - class
+              - vendor
+              - device
           custom:
             - name: "zwave"
               matchOn:

cluster/apps/kube-tools/node-feature-discovery/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - github.com/kubernetes-sigs/node-feature-discovery//deployment/base/nfd-crds?ref=v0.11.2
+  - helm-release.yaml

cluster/apps/kube-tools/reloader/helm-release.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &name reloader
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: *name
+      version: v0.0.118
+      sourceRef:
+        kind: HelmRepository
+        name: stakater-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    fullnameOverride: *name
+    reloader:
+      reloadStrategy: annotations
+      podMonitor:
+        enabled: true
+        namespace: *namespace

cluster/apps/kube-tools/reloader/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml

cluster/apps/kube-tools/system-upgrade/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: system-upgrade
+resources:
+  # renovate: datasource=docker image=rancher/system-upgrade-controller
+  - https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/crd.yaml
+  - system-upgrade-controller

cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/kustomization.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - github.com/rancher/system-upgrade-controller?ref=v0.9.1
+  - plans
+images:
+  - name: rancher/system-upgrade-controller
+    newTag: v0.9.1
+patchesStrategicMerge:
+  # Delete namespace resource
+  - ./system-upgrade-patches.yaml
+  # Add labels
+  - |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: system-upgrade-controller
+      namespace: system-upgrade
+      labels:
+        app.kubernetes.io/name: system-upgrade-controller
+        app.kubernetes.io/instance: system-upgrade-controller

cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/agent.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-agent
+  namespace: system-upgrade
+  labels:
+    k3s-upgrade: agent
+spec:
+  # renovate: datasource=github-releases depName=k3s-io/k3s
+  version: "v1.24.4+k3s1"
+  serviceAccountName: system-upgrade
+  concurrency: 1
+  nodeSelector:
+    matchExpressions:
+      - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist}
+  prepare:
+    image: rancher/k3s-upgrade
+    args: ["prepare", "k3s-server"]
+  upgrade:
+    image: rancher/k3s-upgrade

cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/kustomization.yaml

@@ -2,5 +2,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - controller
-  - plans
+  - server.yaml
+  - agent.yaml

cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/server.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-server
+  namespace: system-upgrade
+  labels:
+    k3s-upgrade: server
+spec:
+  # renovate: datasource=github-releases depName=k3s-io/k3s
+  version: "v1.24.4+k3s1"
+  serviceAccountName: system-upgrade
+  concurrency: 1
+  cordon: true
+  nodeSelector:
+    matchExpressions:
+      - {key: node-role.kubernetes.io/control-plane, operator: Exists}
+  upgrade:
+    image: rancher/k3s-upgrade

cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/system-upgrade-patches.yaml

@@ -0,0 +1,9 @@
+---
+# Namespace should already exist
+# Delete the system-upgrade namespace
+# from the kustomization
+$patch: delete
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: system-upgrade

cluster/apps/kube-tools/vpa/helm-release.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: vpa
+  namespace: default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: vpa
+      version: 1.4.0
+      sourceRef:
+        kind: HelmRepository
+        name: fairwinds-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  values:
+    recommender:
+      enabled: true
+      extraArgs:
+        storage: prometheus
+        prometheus-address: |-
+          http://thanos-query.monitoring.svc.cluster.local:9090
+    updater:
+      enabled: false
+    admissionController:
+      enabled: false

cluster/apps/kube-tools/vpa/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml

cluster/apps/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - namespaces.yaml
   - authentication
   - data
   - databases
@@ -9,6 +10,7 @@ resources:
   - downloaders
   - home-automation
   - kasten-io
+  - kube-tools
   - media
   - monitoring
   - networking

cluster/apps/monitoring/kustomization.yaml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
   - blackbox-exporter
   - grafana
   - healthchecks

cluster/apps/monitoring/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: monitoring

cluster/apps/namespaces.yaml

@@ -0,0 +1,64 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: calico-system
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: default
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flux-system
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-system
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kyverno
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: system-upgrade
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tigera-operator
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    goldilocks.fairwinds.com/enabled: "true"

cluster/charts/descheduler-charts.yaml

@@ -1,10 +1,9 @@
 ---
-apiVersion: source.toolkit.fluxcd.io/v1beta1
+apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
-  name: kubernetes-sigs-descheduler-charts
+  name: descheduler-charts
   namespace: flux-system
 spec:
   interval: 1h
   url: https://kubernetes-sigs.github.io/descheduler
-  timeout: 3m

cluster/charts/fairwinds-charts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: fairwinds-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://charts.fairwinds.com/stable

cluster/charts/kustomization.yaml

@@ -5,8 +5,10 @@ resources:
   - bitnami-charts.yaml
   - bjw-s-charts.yaml
   - cert-manager-webhook-ovh.yaml
+  - descheduler-charts.yaml
   - emxq-charts.yaml
   - external-dns-charts.yaml
+  - fairwinds-charts.yaml
   - gitea-charts.yaml
   - grafana-charts.yaml
   - influxdata-charts.yaml
@@ -15,7 +17,8 @@ resources:
   - k8s-at-home.yaml
   - k8s-gateway-charts.yaml
   - kasten-charts.yaml
-  - kubernetes-sigs-descheduler-charts.yaml
+  - kyverno-charts.yaml
+  - metrics-server-charts.yaml
   - node-feature-discovery.yaml
   - prometheus-community-charts.yaml
   - rook-ceph-charts.yaml

cluster/charts/kyverno-charts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: kyverno-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kyverno.github.io/kyverno/

cluster/charts/metrics-server-charts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: metrics-server-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/metrics-server

cluster/core/flux-system/webhook/github/ingress.yaml

@@ -5,8 +5,7 @@ metadata:
   name: webhook-receiver
   namespace: flux-system
   annotations:
-    external-dns.alpha.kubernetes.io/target: "services.${SECRET_DOMAIN}."
-    external-dns/is-public: "true"
+    external-dns.home.arpa/enabled: "true"
 spec:
   ingressClassName: "nginx"
   rules:

cluster/core/kube-system/descheduler/helm-release.yaml

@@ -1,35 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: descheduler
-  namespace: kube-system
-spec:
-  interval: 5m
-  chart:
-    spec:
-      # renovate: registryUrl=https://kubernetes-sigs.github.io/descheduler
-      chart: descheduler-helm-chart
-      version: 0.19.1
-      sourceRef:
-        kind: HelmRepository
-        name: kubernetes-sigs-descheduler-charts
-        namespace: flux-system
-      interval: 5m
-  values:
-    #schedule: "*/15 * * * *"
-    podAnnotations:
-      botkube.io/disable: "true"
-    deschedulerPolicy:
-      strategies:
-        RemoveDuplicates:
-          enabled: false
-        RemovePodsViolatingNodeAffinity:
-          enabled: true
-          params:
-            nodeAffinityType:
-              - requiredDuringSchedulingIgnoredDuringExecution
-        RemovePodsViolatingInterPodAntiAffinity:
-          enabled: false
-        LowNodeUtilization:
-          enabled: false

cluster/core/kube-system/intel-gpu-plugin/helm-release.yaml

@@ -1,32 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: intel-gpu-plugin
-  namespace: kube-system
-spec:
-  interval: 5m
-  chart:
-    spec:
-      # renovate: registryUrl=https://k8s-at-home.com/charts/
-      chart: intel-gpu-plugin
-      version: 4.4.2
-      sourceRef:
-        kind: HelmRepository
-        name: k8s-at-home-charts
-        namespace: flux-system
-      interval: 5m
-  values:
-    image:
-      repository: ghcr.io/k8s-at-home/intel-gpu-plugin
-      tag: v0.22.0
-      pullPolicy: IfNotPresent
-    affinity:
-      nodeAffinity:
-        requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-            - matchExpressions:
-                - key: feature.node.kubernetes.io/custom-intel-gpu
-                  operator: In
-                  values:
-                    - "true"

cluster/core/kube-system/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: kube-system

cluster/core/kube-system/reloader/helm-release.yaml

@@ -1,25 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: reloader
-  namespace: kube-system
-spec:
-  interval: 5m
-  chart:
-    spec:
-      # renovate: registryUrl=https://stakater.github.io/stakater-charts
-      chart: reloader
-      version: v0.0.118
-      sourceRef:
-        kind: HelmRepository
-        name: stakater-charts
-        namespace: flux-system
-      interval: 5m
-  values:
-    nameOverride: reloader
-    fullnameOverride: reloader
-    reloader:
-      podMonitor:
-        enabled: true
-        namespace: kube-system

cluster/core/kustomization.yaml

@@ -4,7 +4,5 @@ resources:
   - cert-manager
   - flux-system
   - kasten-io
-  - kube-system
   - rook-ceph
-  - system-upgrade
   - storageclasses.yaml

cluster/core/system-upgrade/controller/kustomization.yaml

@@ -1,8 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - github.com/rancher/system-upgrade-controller?ref=v0.9.1
-images:
-  - name: rancher/system-upgrade-controller
-    newTag: v0.9.1

cluster/core/system-upgrade/plans/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - server-plan.yaml
-  - worker-plan.yaml

cluster/core/system-upgrade/plans/server-plan.yaml

@@ -1,25 +0,0 @@
----
-apiVersion: upgrade.cattle.io/v1
-kind: Plan
-metadata:
-  name: k3s-server
-  namespace: system-upgrade
-  labels:
-    k3s-upgrade: server
-spec:
-  concurrency: 1 # Batch size (roughly maps to maximum number of unschedulable nodes)
-  channel: https://update.k3s.io/v1-release/channels/v1.24
-  nodeSelector:
-    matchExpressions:
-      - { key: k3s-upgrade, operator: Exists }
-      - { key: k3s-upgrade, operator: NotIn, values: ["disabled", "false"] }
-      - { key: k3os.io/mode, operator: DoesNotExist }
-      - { key: node-role.kubernetes.io/master, operator: Exists }
-  tolerations:
-    - key: "node-role.kubernetes.io/control-plane"
-      operator: "Exists"
-      effect: "NoSchedule"
-  serviceAccountName: system-upgrade
-  cordon: true
-  upgrade:
-    image: rancher/k3s-upgrade

cluster/core/system-upgrade/plans/worker-plan.yaml

@@ -1,54 +0,0 @@
----
-#
-# Worker plan
-#
-apiVersion: upgrade.cattle.io/v1
-kind: Plan
-metadata:
-  name: k3s-worker
-  namespace: system-upgrade
-  labels:
-    k3s-upgrade: worker
-spec:
-  concurrency: 1
-  channel: https://update.k3s.io/v1-release/channels/v1.24
-  nodeSelector:
-    matchExpressions:
-      #- key: k3s-upgrade
-      #  operator: Exists
-      - key: k3s-upgrade
-        operator: NotIn
-        values:
-          - "disabled"
-          - "false"
-      - key: kubernetes.io/hostname
-        operator: Exists
-      - key: k3os.io/mode
-        operator: DoesNotExist
-      - key: node-role.kubernetes.io/control-plane
-        operator: NotIn
-        values:
-          - "true"
-  serviceAccountName: system-upgrade
-  tolerations:
-    - key: kubernetes.io/arch
-      effect: NoSchedule
-      operator: Equal
-      value: amd64
-    - key: kubernetes.io/arch
-      effect: NoSchedule
-      operator: Equal
-      value: arm64
-    - key: kubernetes.io/arch
-      effect: NoSchedule
-      operator: Equal
-      value: arm
-    - key: arm
-      operator: Exists
-  prepare:
-    image: rancher/k3s-upgrade
-    args:
-      - "prepare"
-      - "k3s-server"
-  upgrade:
-    image: rancher/k3s-upgrade