🔧 use raw chart for jobs

cluster/apps/databases/kustomization.yaml

@@ -4,6 +4,5 @@ kind: Kustomization
 namespace: default
 resources:
   - pgadmin
-  - pgbackups
   - postgres
   - redis

cluster/apps/databases/pgbackups/cron-job.yaml

@@ -1,56 +0,0 @@
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: &app pgbackups
-  namespace: default
-spec:
-  schedule: "@daily"
-  jobTemplate:
-    spec:
-      ttlSecondsAfterFinished: 86400
-      template:
-        spec:
-          automountServiceAccountToken: false
-          restartPolicy: OnFailure
-          containers:
-            - name: pgbackups
-              image: prodrigestivill/postgres-backup-local:14-alpine@sha256:7fe6152197abadd1875c133d474111d4d45643ac045ba64731e3355e78636282
-              env:
-                - name: POSTGRES_HOST
-                  value: postgres-rw.default.svc.cluster.local.
-                - name: POSTGRES_DB
-                  value: "authelia,freshrss,gitea,home_assistant,healthchecks,invidious,joplin,lychee,recipes,sharry,outline,vaultwarden,vikunja,wallabag"
-                - name: POSTGRES_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: postgres-superuser
-                      key: username
-                - name: POSTGRES_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: postgres-superuser
-                      key: password
-                - name: POSTGRES_EXTRA_OPTS
-                  value: "-Z9 --schema=public --blobs"
-                - name: BACKUP_KEEP_DAYS
-                  value: "7"
-                - name: BACKUP_KEEP_WEEKS
-                  value: "4"
-                - name: BACKUP_KEEP_MONTHS
-                  value: "3"
-                - name: HEALTHCHECK_PORT
-                  value: "8080"
-                - name: WEBHOOK_URL
-                  value: http://healthchecks.default.svc.cluster.local.:/ping/${SECRET_HEALTHCHECKS_PING_KEY}/postgresql-backup
-
-              command:
-                - "/backup.sh"
-              volumeMounts:
-                - name: backups
-                  mountPath: /backups
-          volumes:
-            - name: backups
-              nfs:
-                server: "${LOCAL_LAN_TRUENAS}"
-                path: /mnt/storage/backups/postgresql

cluster/apps/databases/postgres/cluster.yaml

@@ -1,34 +0,0 @@
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: postgres
-  namespace: default
-  annotations:
-    kyverno.io/ignore: "true"
-spec:
-  instances: 3
-  primaryUpdateStrategy: unsupervised
-  storage:
-    size: 20Gi
-    storageClass: rook-ceph-block
-  superuserSecret:
-    name: postgres-superuser
-  monitoring:
-    enablePodMonitor: true
-  backup:
-    retentionPolicy: 90d
-    barmanObjectStore:
-      wal:
-        compression: bzip2
-        maxParallel: 8
-      destinationPath: s3://postgresql/
-      endpointURL: https://truenas.${SECRET_DOMAIN}:9000
-      serverName: postgres
-      s3Credentials:
-        accessKeyId:
-          name: postgres-minio
-          key: MINIO_ACCESS_KEY
-        secretAccessKey:
-          name: postgres-minio
-          key: MINIO_SECRET_KEY

cluster/apps/databases/postgres/cluster/helm-release.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: postgres-cluster
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: postgres
+      namespace: default
+    - name: rook-ceph-cluster
+      namespace: rook-ceph
+  values:
+    resources:
+      - apiVersion: postgresql.cnpg.io/v1
+        kind: Cluster
+        metadata:
+          name: postgres
+          namespace: *namespace
+          annotations:
+            kyverno.io/ignore: "true"
+        spec:
+          instances: 3
+          primaryUpdateStrategy: unsupervised
+          storage:
+            size: 20Gi
+            storageClass: rook-ceph-block
+          superuserSecret:
+            name: postgres-superuser
+          monitoring:
+            enablePodMonitor: true
+          backup:
+            retentionPolicy: 30d
+            barmanObjectStore:
+              wal:
+                compression: bzip2
+                maxParallel: 8
+              destinationPath: s3://postgresql/
+              endpointURL: https://truenas.${SECRET_DOMAIN}:9000
+              serverName: postgres
+              s3Credentials:
+                accessKeyId:
+                  name: postgres-minio
+                  key: MINIO_ACCESS_KEY
+                secretAccessKey:
+                  name: postgres-minio
+                  key: MINIO_SECRET_KEY

cluster/apps/databases/postgres/cluster/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret.sops.yaml
+  - helm-release.yaml

cluster/apps/databases/postgres/cluster/secret.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/basic-auth
+metadata:
+    name: postgres-superuser
+    namespace: default
+stringData:
+    username: ENC[AES256_GCM,data:oMwUm7mTJ3U=,iv:hfa6GmA8uFC1gPs7Z0wAaddOhVeHu8FmANOd9n/fLok=,tag:FIv7VhkHlVLq4Q+k7N2DDw==,type:str]
+    password: ENC[AES256_GCM,data:LCUuhRW3wjkeVQgefTuh9Q==,iv:07R0ZUrLQe8jPZo3wFn/15fXg8yc/pa+a03tWkSrjjM=,tag:0YoG2EZ3JbihlY98ay/5eg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1hhurqwmfvl9m3vh3hk8urulfzcdsrep2ax2neazqt435yhpamu3qj20asg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQand1M1U2SytHclJSN1I3
+            NzdvdjZMQnJPSW9GUXo1SkZ1elRVY1NvK0FJClpiVk9JVWxHSlIwSXZDSWRoOXI4
+            YkxVeDR5V09OTS92YmpMeUl2a1QyRlUKLS0tIG9iNGJlaDQ3UW1uelFla0cySXRC
+            SzhQOGRzNnYzcEVjVG0rOUt1T1ZJQkkKtbXybUgBFr69GvBmo8+7J1xrtxJ7y1wo
+            ZhV6dzuxc2QSd3o9A6f9J/wg9DHtBHviK5nP0K/edHth9darJw/3Eg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-10-25T23:37:50Z"
+    mac: ENC[AES256_GCM,data:aU5GLUX3Tml3tRZUzRP451X5oeUSEpB2QFp7ys8pnKlskDidWwwy3gCCTeG0gjsmJbYiZqZFS0qnYe5brT1b9gJgQVLTgVA8xcoXMFJnGQfHm+kmqBxfYR2wPyCzE3T/J4/2e01oITuVS5RKtc3/w1L2en8DwttcBBaezh3vRRM=,iv:a11Hm95soVPiALzZSHMkKx+XEdq7PPmVysfhXHY0+pw=,tag:ITVZQw2WSmD9rmU/cSto4w==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.7.3

cluster/apps/databases/postgres/external-backup/helm-release.yaml

@@ -0,0 +1,83 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app postgres-external-backup
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: postgres
+      namespace: default
+  values:
+    resources:
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: *app
+          namespace: *namespace
+        spec:
+          schedule: "@daily"
+          jobTemplate:
+            spec:
+              ttlSecondsAfterFinished: 86400
+              template:
+                spec:
+                  automountServiceAccountToken: false
+                  restartPolicy: OnFailure
+                  containers:
+                    - name: *app
+                      image: prodrigestivill/postgres-backup-local:14-alpine@sha256:7fe6152197abadd1875c133d474111d4d45643ac045ba64731e3355e78636282
+                      env:
+                        - name: POSTGRES_HOST
+                          value: postgres-rw.default.svc.cluster.local.
+                        - name: POSTGRES_DB
+                          value: "authelia,freshrss,gitea,home_assistant,healthchecks,invidious,joplin,lychee,recipes,sharry,outline,vaultwarden,vikunja,wallabag"
+                        - name: POSTGRES_USER
+                          valueFrom:
+                            secretKeyRef:
+                              name: postgres-superuser
+                              key: username
+                        - name: POSTGRES_PASSWORD
+                          valueFrom:
+                            secretKeyRef:
+                              name: postgres-superuser
+                              key: password
+                        - name: POSTGRES_EXTRA_OPTS
+                          value: "-Z9 --schema=public --blobs"
+                        - name: BACKUP_KEEP_DAYS
+                          value: "7"
+                        - name: BACKUP_KEEP_WEEKS
+                          value: "4"
+                        - name: BACKUP_KEEP_MONTHS
+                          value: "3"
+                        - name: HEALTHCHECK_PORT
+                          value: "8080"
+                        - name: WEBHOOK_URL
+                          value: http://healthchecks.default.svc.cluster.local.:/ping/${SECRET_HEALTHCHECKS_PING_KEY}/postgresql-backup
+
+                      command:
+                        - "/backup.sh"
+                      volumeMounts:
+                        - name: backups
+                          mountPath: /backups
+                  volumes:
+                    - name: backups
+                      nfs:
+                        server: "${LOCAL_LAN_TRUENAS}"
+                        path: /mnt/storage/backups/postgresql

cluster/apps/databases/postgres/external-backup/kustomization.yaml

@@ -1,5 +1,4 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - cron-job.yaml
+  - helm-release.yaml

cluster/apps/databases/postgres/kustomization.yaml

@@ -2,10 +2,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - secret.sops.yaml
   - helm-release.yaml
-  - cluster.yaml
+  - cluster
-  - scheduled-backup.yaml
+  - external-backup
+  - scheduled-backup
 configMapGenerator:
   - name: cloudnative-pg-dashboard
     files:

cluster/apps/databases/postgres/scheduled-backup.yaml

@@ -1,11 +0,0 @@
-apiVersion: postgresql.cnpg.io/v1
-kind: ScheduledBackup
-metadata:
-  name: postgres
-  namespace: default
-spec:
-  schedule: "@daily"
-  immediate: true
-  backupOwnerReference: self
-  cluster:
-    name: postgres

cluster/apps/databases/postgres/scheduled-backup/helm-release.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: postgres-scheduled-backup
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: postgres
+      namespace: default
+    - name: rook-ceph-cluster
+      namespace: rook-ceph
+  values:
+    resources:
+      - apiVersion: postgresql.cnpg.io/v1
+        kind: ScheduledBackup
+        metadata:
+          name: postgres
+          namespace: *namespace
+        spec:
+          schedule: "@daily"
+          immediate: true
+          backupOwnerReference: self
+          cluster:
+            name: postgres

cluster/apps/databases/postgres/scheduled-backup/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret.sops.yaml
+  - helm-release.yaml

cluster/apps/databases/postgres/scheduled-backup/secret.sops.yaml

@@ -0,0 +1,31 @@
+kind: Secret
+apiVersion: v1
+type: Opaque
+metadata:
+    name: postgres-minio
+    namespace: default
+    labels:
+        k8s.enterprisedb.io/reload: "true"
+stringData:
+    MINIO_ACCESS_KEY: ENC[AES256_GCM,data:lEOKspQaoN5FxOGSnpQuTAzzHrI=,iv:VJQAWK8Sia/wL4iAdpir5fJxBLP1fDQWqj5pBDO6x/g=,tag:5Jf612CStm7NcW1YdrOq1A==,type:str]
+    MINIO_SECRET_KEY: ENC[AES256_GCM,data:Saad8zdhNfJdCDM/3cwVAtp/Cx8F0R4AFERJA3xT7ZC7M0GptDVaGg==,iv:DnmbB6VCRa2itDLAYwGL3LkTBQlf4sVwu1O5+ZmuukQ=,tag:fG6XMj/rC3moGKVZJn9PBA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1hhurqwmfvl9m3vh3hk8urulfzcdsrep2ax2neazqt435yhpamu3qj20asg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQand1M1U2SytHclJSN1I3
+            NzdvdjZMQnJPSW9GUXo1SkZ1elRVY1NvK0FJClpiVk9JVWxHSlIwSXZDSWRoOXI4
+            YkxVeDR5V09OTS92YmpMeUl2a1QyRlUKLS0tIG9iNGJlaDQ3UW1uelFla0cySXRC
+            SzhQOGRzNnYzcEVjVG0rOUt1T1ZJQkkKtbXybUgBFr69GvBmo8+7J1xrtxJ7y1wo
+            ZhV6dzuxc2QSd3o9A6f9J/wg9DHtBHviK5nP0K/edHth9darJw/3Eg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-10-25T23:37:42Z"
+    mac: ENC[AES256_GCM,data:VZ5+kUZsCJxiWV7JS+Enhi0yNJ6m+Oi5IurYNxI0gb2+CqENqn4uvOSNMgKTZAc3d/stuI5OGdBbRJo0aBu0hZ950cgbGV6gfEbzzTO9HRstgAwqnEZHj6DPRLcXkCs0jP1p2p0WICe2HZ113C2aN3MjP47J1Jau3yaJlGOsOuU=,iv:EaxUx+ivqYgBm1wUXsCscoJt7x6+3pSM0QZY8h9eI6U=,tag:Q5ix3VW7C2rgm2R3AMDuDA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.7.3

cluster/apps/databases/postgres/secret.sops.yaml

@@ -1,61 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: kubernetes.io/basic-auth
-metadata:
-    name: postgres-superuser
-    namespace: default
-stringData:
-    username: ENC[AES256_GCM,data:oMwUm7mTJ3U=,iv:hfa6GmA8uFC1gPs7Z0wAaddOhVeHu8FmANOd9n/fLok=,tag:FIv7VhkHlVLq4Q+k7N2DDw==,type:str]
-    password: ENC[AES256_GCM,data:LCUuhRW3wjkeVQgefTuh9Q==,iv:07R0ZUrLQe8jPZo3wFn/15fXg8yc/pa+a03tWkSrjjM=,tag:0YoG2EZ3JbihlY98ay/5eg==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1hhurqwmfvl9m3vh3hk8urulfzcdsrep2ax2neazqt435yhpamu3qj20asg
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQand1M1U2SytHclJSN1I3
-            NzdvdjZMQnJPSW9GUXo1SkZ1elRVY1NvK0FJClpiVk9JVWxHSlIwSXZDSWRoOXI4
-            YkxVeDR5V09OTS92YmpMeUl2a1QyRlUKLS0tIG9iNGJlaDQ3UW1uelFla0cySXRC
-            SzhQOGRzNnYzcEVjVG0rOUt1T1ZJQkkKtbXybUgBFr69GvBmo8+7J1xrtxJ7y1wo
-            ZhV6dzuxc2QSd3o9A6f9J/wg9DHtBHviK5nP0K/edHth9darJw/3Eg==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2022-09-14T11:46:06Z"
-    mac: ENC[AES256_GCM,data:+FQLnaq6xHe/NwKGvBQBDcIyJmdHWi612OhFucMOSfNBIDs70oUV96zay2qg3Ish0O4hTmUY8T4akVnRJj6hAYR/BY0yQ6v0fZAaVMc0AjPEi/kDuCIkvet3FOraU3hdL1sKE7zd+h8Xohen0n7dYsYXfH9ZN7QkPQx6Dn+HQcU=,iv:Wou+7naYwOc+5iw+Gn6BQm9Hmxg8Zycrab+LJZti5rw=,tag:M7t+PDAB50Y2zDxfP1GRag==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.7.3
----
-kind: Secret
-apiVersion: v1
-type: Opaque
-metadata:
-    name: postgres-minio
-    namespace: default
-    labels:
-        k8s.enterprisedb.io/reload: "true"
-stringData:
-    MINIO_ACCESS_KEY: ENC[AES256_GCM,data:lEOKspQaoN5FxOGSnpQuTAzzHrI=,iv:VJQAWK8Sia/wL4iAdpir5fJxBLP1fDQWqj5pBDO6x/g=,tag:5Jf612CStm7NcW1YdrOq1A==,type:str]
-    MINIO_SECRET_KEY: ENC[AES256_GCM,data:Saad8zdhNfJdCDM/3cwVAtp/Cx8F0R4AFERJA3xT7ZC7M0GptDVaGg==,iv:DnmbB6VCRa2itDLAYwGL3LkTBQlf4sVwu1O5+ZmuukQ=,tag:fG6XMj/rC3moGKVZJn9PBA==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1hhurqwmfvl9m3vh3hk8urulfzcdsrep2ax2neazqt435yhpamu3qj20asg
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQand1M1U2SytHclJSN1I3
-            NzdvdjZMQnJPSW9GUXo1SkZ1elRVY1NvK0FJClpiVk9JVWxHSlIwSXZDSWRoOXI4
-            YkxVeDR5V09OTS92YmpMeUl2a1QyRlUKLS0tIG9iNGJlaDQ3UW1uelFla0cySXRC
-            SzhQOGRzNnYzcEVjVG0rOUt1T1ZJQkkKtbXybUgBFr69GvBmo8+7J1xrtxJ7y1wo
-            ZhV6dzuxc2QSd3o9A6f9J/wg9DHtBHviK5nP0K/edHth9darJw/3Eg==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2022-09-14T11:46:06Z"
-    mac: ENC[AES256_GCM,data:+FQLnaq6xHe/NwKGvBQBDcIyJmdHWi612OhFucMOSfNBIDs70oUV96zay2qg3Ish0O4hTmUY8T4akVnRJj6hAYR/BY0yQ6v0fZAaVMc0AjPEi/kDuCIkvet3FOraU3hdL1sKE7zd+h8Xohen0n7dYsYXfH9ZN7QkPQx6Dn+HQcU=,iv:Wou+7naYwOc+5iw+Gn6BQm9Hmxg8Zycrab+LJZti5rw=,tag:M7t+PDAB50Y2zDxfP1GRag==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.7.3

cluster/apps/development/gitea/backup-job.yaml

@@ -1,92 +0,0 @@
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: gitea-repositories-backup
-  namespace: default
-spec:
-  schedule: "@daily"
-  jobTemplate:
-    spec:
-      template:
-        metadata:
-          name: gitea-repositories-backup
-        spec:
-          containers:
-            - name: gitea-repositories-backup
-              image: ghcr.io/auricom/kubectl:v1.25.0@sha256:75d43a3131e25f10139174e77e689eafb1b40ed9d9094d5c8c96eba5571aeefd
-              imagePullPolicy: IfNotPresent
-              env:
-                - name: ENV_GITEA_API_TOKEN
-                  valueFrom:
-                    secretKeyRef:
-                      name: gitea-config
-                      key: apiToken
-              command:
-                - "/bin/bash"
-                - "-c"
-                - |
-                  #!/bin/bash
-
-                  set -o nounset
-                  set -o errexit
-
-                  mkdir -p ~/.ssh
-                  cp /opt/id_rsa ~/.ssh/id_rsa
-                  chmod 600 ~/.ssh/id_rsa
-
-                  ssh -o StrictHostKeyChecking=no homelab@${LOCAL_LAN_TRUENAS} << 'EOF'
-
-                    WORK_DIR="/mnt/storage/backups/apps/gitea"
-
-                    ORGANISATIONS=$(curl --silent --location --request GET "https://gitea.${SECRET_CLUSTER_DOMAIN}/api/v1/orgs" --header "Authorization: Bearer ${ENV_GITEA_API_TOKEN}" | jq --raw-output .[].username)
-                    ORGANISATIONS+=" auricom"
-
-                    for org in $ORGANISATIONS
-                    do
-                        mkdir -p $WORK_DIR/$org
-                        if [ $org == "auricom" ]; then
-                            keyword="users"
-                        else
-                            keyword="orgs"
-                        fi
-                        REPOSITORIES=$(curl --silent --location --request GET "https://gitea.${SECRET_CLUSTER_DOMAIN}/api/v1/$keyword/$org/repos?limit=1000" --header "Authorization: Bearer ${ENV_GITEA_API_TOKEN}" | jq --raw-output .[].name)
-                        for repo in $REPOSITORIES
-                        do
-                            if [ -d "$WORK_DIR/$org/$repo" ]; then
-                                echo "INFO: pull $org/$repo..."
-                                cd $WORK_DIR/$org/$repo
-                                git remote show origin -n | grep -c main &> /dev/null && MAIN_BRANCH="main" || MAIN_BRANCH="master"
-                                git fetch --all
-                                test $? -ne 0 && exit 1
-                                git reset --hard origin/$MAIN_BRANCH
-                                test $? -ne 0 && exit 1
-                                git pull origin $MAIN_BRANCH
-                                test $? -ne 0 && exit 1
-                                echo "INFO: clean $org/$repo..."
-                                git fetch --prune
-                                for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}')
-                                do
-                                  git branch -D $branch
-                                done
-                            else
-                                echo "INFO: clone $org/$repo..."
-                                cd $WORK_DIR/$org
-                                git clone git@gitea.${SECRET_DOMAIN}:$org/$repo.git
-                                test $? -ne 0 && exit 1
-                            fi
-                        done
-                    done
-                    echo "INFO: Backup done"
-                  EOF
-
-                  curl -m 10 --retry 5 http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-gitea-repositories-backup
-              volumeMounts:
-                - name: secret
-                  mountPath: /opt/id_rsa
-                  subPath: deployment_rsa_priv_key
-          volumes:
-            - name: secret
-              secret:
-                secretName: gitea-config
-          restartPolicy: Never

cluster/apps/development/gitea/external-backup/helm-release.yaml

@@ -0,0 +1,119 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app gitea-external-backup
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: gitea
+      namespace: default
+  values:
+    resources:
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: *app
+          namespace: *namespace
+        spec:
+          schedule: "@daily"
+          jobTemplate:
+            spec:
+              template:
+                metadata:
+                  name: *app
+                spec:
+                  containers:
+                    - name: *app
+                      image: ghcr.io/auricom/kubectl:v1.25.0@sha256:75d43a3131e25f10139174e77e689eafb1b40ed9d9094d5c8c96eba5571aeefd
+                      imagePullPolicy: IfNotPresent
+                      env:
+                        - name: ENV_GITEA_API_TOKEN
+                          valueFrom:
+                            secretKeyRef:
+                              name: gitea-config
+                              key: apiToken
+                      command:
+                        - "/bin/bash"
+                        - "-c"
+                        - |
+                          #!/bin/bash
+
+                          set -o nounset
+                          set -o errexit
+
+                          mkdir -p ~/.ssh
+                          cp /opt/id_rsa ~/.ssh/id_rsa
+                          chmod 600 ~/.ssh/id_rsa
+
+                          ssh -o StrictHostKeyChecking=no homelab@${LOCAL_LAN_TRUENAS} << 'EOF'
+
+                            WORK_DIR="/mnt/storage/backups/apps/gitea"
+
+                            ORGANISATIONS=$(curl --silent --location --request GET "https://gitea.${SECRET_CLUSTER_DOMAIN}/api/v1/orgs" --header "Authorization: Bearer ${ENV_GITEA_API_TOKEN}" | jq --raw-output .[].username)
+                            ORGANISATIONS+=" auricom"
+
+                            for org in $ORGANISATIONS
+                            do
+                                mkdir -p $WORK_DIR/$org
+                                if [ $org == "auricom" ]; then
+                                    keyword="users"
+                                else
+                                    keyword="orgs"
+                                fi
+                                REPOSITORIES=$(curl --silent --location --request GET "https://gitea.${SECRET_CLUSTER_DOMAIN}/api/v1/$keyword/$org/repos?limit=1000" --header "Authorization: Bearer ${ENV_GITEA_API_TOKEN}" | jq --raw-output .[].name)
+                                for repo in $REPOSITORIES
+                                do
+                                    if [ -d "$WORK_DIR/$org/$repo" ]; then
+                                        echo "INFO: pull $org/$repo..."
+                                        cd $WORK_DIR/$org/$repo
+                                        git remote show origin -n | grep -c main &> /dev/null && MAIN_BRANCH="main" || MAIN_BRANCH="master"
+                                        git fetch --all
+                                        test $? -ne 0 && exit 1
+                                        git reset --hard origin/$MAIN_BRANCH
+                                        test $? -ne 0 && exit 1
+                                        git pull origin $MAIN_BRANCH
+                                        test $? -ne 0 && exit 1
+                                        echo "INFO: clean $org/$repo..."
+                                        git fetch --prune
+                                        for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}')
+                                        do
+                                          git branch -D $branch
+                                        done
+                                    else
+                                        echo "INFO: clone $org/$repo..."
+                                        cd $WORK_DIR/$org
+                                        git clone git@gitea.${SECRET_DOMAIN}:$org/$repo.git
+                                        test $? -ne 0 && exit 1
+                                    fi
+                                done
+                            done
+                            echo "INFO: Backup done"
+                          EOF
+
+                          curl -m 10 --retry 5 http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-gitea-repositories-backup
+                      volumeMounts:
+                        - name: secret
+                          mountPath: /opt/id_rsa
+                          subPath: deployment_rsa_priv_key
+                  volumes:
+                    - name: secret
+                      secret:
+                        secretName: gitea-config
+                  restartPolicy: Never

cluster/apps/development/gitea/external-backup/kustomization.yaml

@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - cron-job.yaml
+  - helm-release.yaml

cluster/apps/development/gitea/kustomization.yaml

@@ -4,4 +4,4 @@ resources:
   - secret.sops.yaml
   - volume.yaml
   - helm-release.yaml
-  - backup-job.yaml
+  - external-backup

cluster/apps/downloaders/kustomization.yaml

@@ -6,5 +6,4 @@ resources:
   - flood
   - pyload
   - qbittorrent
-  - qbittorrent-jobs
   - sabnzbd

cluster/apps/downloaders/qbittorrent-jobs/cron-job.yaml

@@ -1,41 +0,0 @@
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: qbittorrent-upgrade-p2pblocklist
-  namespace: default
-spec:
-  schedule: "@daily"
-  jobTemplate:
-    spec:
-      template:
-        metadata:
-          name: qbittorrent-upgrade-p2pblocklist
-        spec:
-          serviceAccountName: jobs
-          containers:
-            - name: qbittorrent-upgrade-p2pblocklist
-              image: ghcr.io/auricom/kubectl:v1.25.0@sha256:75d43a3131e25f10139174e77e689eafb1b40ed9d9094d5c8c96eba5571aeefd
-              imagePullPolicy: IfNotPresent
-              command:
-                - "/bin/bash"
-                - "-c"
-                - |
-                  #!/bin/bash
-
-                  set -o errexit
-                  set -o nounset
-
-                  curl --location https://github.com/DavidMoore/ipfilter/releases/download/lists/ipfilter.dat.gz --output /tmp/ipfilter.dat.gz
-                  gunzip /tmp/ipfilter.dat.gz
-                  result=$(kubectl get pod --selector app.kubernetes.io/name=qbittorrent --output custom-columns=:metadata.name --namespace default)
-                  QBITTORRENT_POD=$(echo $result | awk '{ print $NF }')
-                  echo $QBITTORRENT_POD | grep qbittorrent
-                  if [[ $(echo $QBITTORRENT_POD | grep qbittorrent) ]]; then
-                    kubectl cp /tmp/ipfilter.dat default/$QBITTORRENT_POD:/config/ipfilter.dat
-                    kubectl rollout restart deployment qbittorrent --namespace default && curl -m 10 --retry 5 http://healthchecks.default.svc.cluster.local.:/ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-qbittorrent-p2pblocklist
-                  else
-                    echo "qbittorrent deployment not found"
-                    exit 1
-                  fi
-          restartPolicy: Never

cluster/apps/downloaders/qbittorrent/upgrade-p2pblocklist/helm-release.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app qbittorrent-upgrade-p2pblocklist
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: qbittorrent
+      namespace: default
+  values:
+    resources:
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: *app
+          namespace: *namespace
+        spec:
+          schedule: "@daily"
+          jobTemplate:
+            spec:
+              template:
+                metadata:
+                  name: *app
+                spec:
+                  serviceAccountName: jobs
+                  containers:
+                    - name: *app
+                      image: ghcr.io/auricom/kubectl:v1.25.0@sha256:75d43a3131e25f10139174e77e689eafb1b40ed9d9094d5c8c96eba5571aeefd
+                      imagePullPolicy: IfNotPresent
+                      command:
+                        - "/bin/bash"
+                        - "-c"
+                        - |
+                          #!/bin/bash
+
+                          set -o errexit
+                          set -o nounset
+
+                          curl --location https://github.com/DavidMoore/ipfilter/releases/download/lists/ipfilter.dat.gz --output /tmp/ipfilter.dat.gz
+                          gunzip /tmp/ipfilter.dat.gz
+                          result=$(kubectl get pod --selector app.kubernetes.io/name=qbittorrent --output custom-columns=:metadata.name --namespace default)
+                          QBITTORRENT_POD=$(echo $result | awk '{ print $NF }')
+                          echo $QBITTORRENT_POD | grep qbittorrent
+                          if [[ $(echo $QBITTORRENT_POD | grep qbittorrent) ]]; then
+                            kubectl cp /tmp/ipfilter.dat default/$QBITTORRENT_POD:/config/ipfilter.dat
+                            kubectl rollout restart deployment qbittorrent --namespace default && curl -m 10 --retry 5 http://healthchecks.default.svc.cluster.local.:/ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-qbittorrent-p2pblocklist
+                          else
+                            echo "qbittorrent deployment not found"
+                            exit 1
+                          fi
+                  restartPolicy: Never

cluster/apps/downloaders/qbittorrent/upgrade-p2pblocklist/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml
+  - upgrade-p2pblocklist

cluster/apps/media-automation/recyclarr/cron-job.yaml

@@ -1,72 +0,0 @@
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: &app recyclarr
-  namespace: default
-spec:
-  schedule: "@daily"
-  jobTemplate:
-    spec:
-      ttlSecondsAfterFinished: 86400
-      template:
-        spec:
-          automountServiceAccountToken: false
-          restartPolicy: OnFailure
-          initContainers:
-            - name: render-configs
-              image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
-              envFrom:
-                - secretRef:
-                    name: *app
-              command:
-                - "/bin/bash"
-                - -c
-              args:
-                - "envsubst < /config/recyclarr.yaml > /shared/recyclarr.yaml"
-              volumeMounts:
-                - name: config
-                  mountPath: /config
-                - name: shared
-                  mountPath: /shared
-          containers:
-            - name: sonarr
-              image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
-              env:
-                - name: TZ
-                  value: "${TIMEZONE}"
-              command:
-                - "/bin/bash"
-                - "-c"
-                - |
-                  #!/bin/bash
-
-                  /app/recyclarr sonarr --config /config/recyclarr.yaml && curl -fsS -m 10 --retry 5 -o /dev/null http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-recyclarr-sonarr
-              volumeMounts:
-                - name: shared
-                  mountPath: /config/recyclarr.yaml
-                  subPath: recyclarr.yaml
-                  readOnly: true
-            - name: radarrs
-              image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
-              env:
-                - name: TZ
-                  value: "${TIMEZONE}"
-              command:
-                - "/bin/bash"
-                - "-c"
-                - |
-                  #!/bin/bash
-
-                  /app/recyclarr radarr --config /config/recyclarr.yaml && curl -fsS -m 10 --retry 5 -o /dev/null http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-recyclarr-radarr
-              volumeMounts:
-                - name: shared
-                  mountPath: /config/recyclarr.yaml
-                  subPath: recyclarr.yaml
-                  readOnly: true
-          volumes:
-            - name: config
-              configMap:
-                name: *app
-            - name: shared
-              emptyDir: {}

cluster/apps/media-automation/recyclarr/helm-release.yaml

@@ -0,0 +1,101 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app recyclarr
+  namespace: &namespace default
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: sonarr
+      namespace: default
+    - name: radarr
+      namespace: default
+  values:
+    resources:
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: *app
+          namespace: *namespace
+        spec:
+          schedule: "@daily"
+          jobTemplate:
+            spec:
+              ttlSecondsAfterFinished: 86400
+              template:
+                spec:
+                  automountServiceAccountToken: false
+                  restartPolicy: OnFailure
+                  initContainers:
+                    - name: render-configs
+                      image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
+                      envFrom:
+                        - secretRef:
+                            name: *app
+                      command:
+                        - "/bin/bash"
+                        - -c
+                      args:
+                        - "envsubst < /config/recyclarr.yaml > /shared/recyclarr.yaml"
+                      volumeMounts:
+                        - name: config
+                          mountPath: /config
+                        - name: shared
+                          mountPath: /shared
+                  containers:
+                    - name: sonarr
+                      image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
+                      env:
+                        - name: TZ
+                          value: "${TIMEZONE}"
+                      command:
+                        - "/bin/bash"
+                        - "-c"
+                        - |
+                          #!/bin/bash
+
+                          /app/recyclarr sonarr --config /config/recyclarr.yaml && curl -fsS -m 10 --retry 5 -o /dev/null http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-recyclarr-sonarr
+                      volumeMounts:
+                        - name: shared
+                          mountPath: /config/recyclarr.yaml
+                          subPath: recyclarr.yaml
+                          readOnly: true
+                    - name: radarrs
+                      image: ghcr.io/onedr0p/recyclarr:2.6.1@sha256:f39ffcce0d5b75c4b60299c6a4177c4d247ea8678bea1ce2fb7ada14e4206c9f
+                      env:
+                        - name: TZ
+                          value: "${TIMEZONE}"
+                      command:
+                        - "/bin/bash"
+                        - "-c"
+                        - |
+                          #!/bin/bash
+
+                          /app/recyclarr radarr --config /config/recyclarr.yaml && curl -fsS -m 10 --retry 5 -o /dev/null http://healthchecks.default.svc.cluster.local./ping/${SECRET_HEALTHCHECKS_PING_KEY}/k3s-recyclarr-radarr
+                      volumeMounts:
+                        - name: shared
+                          mountPath: /config/recyclarr.yaml
+                          subPath: recyclarr.yaml
+                          readOnly: true
+                  volumes:
+                    - name: config
+                      configMap:
+                        name: *app
+                    - name: shared
+                      emptyDir: {}

cluster/apps/media-automation/recyclarr/kustomization.yaml

@@ -3,12 +3,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - secret.sops.yaml
-  - cron-job.yaml
+  - helm-release.yaml
 namespace: default
 configMapGenerator:
   - name: recyclarr
     files:
-      - recyclarr.yaml
+      - config/recyclarr.yaml
 generatorOptions:
   disableNameSuffixHash: true
   annotations: