♻️ migrate postgresql to truenas jail + minio https

2025-09-17 18:24:14 +02:00 · 2024-01-13 17:47:18 +01:00
parent badd042d50
commit 7fd3c78db8
111 changed files with 785 additions and 266 deletions
--- a/kubernetes/apps/default/homelab/ks.yaml
+++ b/kubernetes/apps/default/homelab/ks.yaml
@@ -3,7 +3,7 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: cluster-apps-homnelab-minio
+  name: cluster-apps-homelab-minio
  namespace: flux-system
 spec:
  path: ./kubernetes/apps/default/homelab/minio
@@ -21,7 +21,7 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: cluster-apps-homnelab-opnsense
+  name: cluster-apps-homelab-opnsense
  namespace: flux-system
 spec:
  path: ./kubernetes/apps/default/homelab/opnsense
@@ -39,7 +39,7 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: cluster-apps-homnelab-truenas
+  name: cluster-apps-homelab-truenas
  namespace: flux-system
 spec:
  path: ./kubernetes/apps/default/homelab/truenas
--- a/kubernetes/apps/default/homelab/minio/backup/rclone.conf
+++ b/kubernetes/apps/default/homelab/minio/backup/rclone.conf
@@ -3,7 +3,7 @@ type = s3
 provider = Minio
 access_key_id = __RCLONE_ACCESS_ID__
 secret_access_key = __RCLONE_SECRET_KEY__
-endpoint = http://minio.${SECRET_DOMAIN}:9000
+endpoint = https://minio.${SECRET_DOMAIN}:9000
 acl = private

 [gdrive-homelab-backups]
--- a/kubernetes/apps/default/homelab/opnsense/backup/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/opnsense/backup/helmrelease.yaml
@@ -41,7 +41,7 @@ spec:
            command: ["/bin/bash", "/app/opnsense-backup.sh"]
            env:
              OPNSENSE_URL: "https://opnsense.${SECRET_DOMAIN}"
-              S3_URL: "http://minio.${SECRET_DOMAIN}:9000"
+              S3_URL: "https://minio.${SECRET_DOMAIN}:9000"
            envFrom:
              - secretRef:
                  name: homelab-opnsense-secret
--- a/kubernetes/apps/default/homelab/truenas/backup/truenas-backup.sh
+++ b/kubernetes/apps/default/homelab/truenas/backup/truenas-backup.sh
@@ -44,7 +44,7 @@ curl -fsSL \
    -H "Date: ${http_request_date}" \
    -H "Content-Type: ${http_content_type}" \
    -H "Authorization: AWS ${AWS_ACCESS_KEY_ID}:${http_signature}" \
-    "http://minio.${SECRET_DOMAIN}:9000/${http_filepath}"
+    "https://minio.${SECRET_DOMAIN}:9000/${http_filepath}"

 rm /tmp/backup-*.tar

--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/helmrelease.yaml
@@ -42,7 +42,8 @@ spec:
            env:
              HOSTNAME: truenas
              TRUENAS_HOME: /mnt/storage/home/homelab
-              CERTS_DEPLOY_S3_ENABLED: "True"
+              CERTS_DEPLOY_MINIO_ENABLED: "True"
+              CERTS_DEPLOY_POSTGRESQL_ENABLED: "True"
            envFrom: &envFrom
              - secretRef:
                  name: &secret homelab-truenas-secret
@@ -54,7 +55,8 @@ spec:
            env:
              HOSTNAME: truenas-remote
              TRUENAS_HOME: /mnt/vol1/home/homelab
-              CERTS_DEPLOY_S3_ENABLED: "False"
+              CERTS_DEPLOY_MINIO_ENABLED: "False"
+              CERTS_DEPLOY_POSTGRESQL_ENABLED: "False"
            envFrom: *envFrom
    service:
      main:
--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.sh
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.sh
@@ -12,21 +12,22 @@ if [ "${HOSTNAME}" == "truenas" ]; then
 elif [ "${HOSTNAME}" == "truenas-remote" ]; then
    printf -v truenas_api_key %q "$TRUENAS_REMOTE_API_KEY"
 fi
-printf -v cert_deploy_s3_enabled_str %q "$CERTS_DEPLOY_S3_ENABLED"
+printf -v cert_deploy_minio_enabled_str %q "$CERTS_DEPLOY_MINIO_ENABLED"
+printf -v cert_deploy_postgresql_enabled_str %q "$CERTS_DEPLOY_POSTGRESQL_ENABLED"
 printf -v pushover_api_token_str %q "$PUSHOVER_API_TOKEN"
 printf -v pushover_user_key_str %q "$PUSHOVER_USER_KEY"
 printf -v secret_domain_str %q "$SECRET_DOMAIN"

 scp -o StrictHostKeyChecking=no /app/truenas-certs-deploy.py homelab@${HOSTNAME}.${SECRET_DOMAIN}:${TRUENAS_HOME}/scripts/certificates_deploy.py

-ssh -o StrictHostKeyChecking=no homelab@${HOSTNAME}.${SECRET_DOMAIN} "/bin/bash -s $truenas_api_key $cert_deploy_s3_enabled_str $pushover_api_token_str $pushover_user_key_str $secret_domain_str" << 'EOF'
+ssh -o StrictHostKeyChecking=no homelab@${HOSTNAME}.${SECRET_DOMAIN} "/bin/bash -s $truenas_api_key $cert_deploy_minio_enabled_str $cert_deploy_postgresql_enabled_str $pushover_api_token_str $pushover_user_key_str $secret_domain_str" << 'EOF'

 set -o nounset
 set -o errexit

-PUSHOVER_API_TOKEN=$3
-PUSHOVER_USER_KEY=$4
-SECRET_DOMAIN=$5
+PUSHOVER_API_TOKEN=$4
+PUSHOVER_USER_KEY=$5
+SECRET_DOMAIN=$6

 # Variables
 TARGET=$(hostname)
@@ -38,8 +39,13 @@ export CERTS_DEPLOY_API_KEY=$1
 export CERTS_DEPLOY_PRIVATE_KEY_PATH=${CERTIFICATE_PATH}/key.pem
 export CERTS_DEPLOY_FULLCHAIN_PATH=${CERTIFICATE_PATH}/fullchain.pem
 if [ "$2" == "True" ]; then
-    export CERTS_DEPLOY_S3_ENABLED=$2
+    export CERTS_DEPLOY_MINIO_ENABLED=$2
 fi
+CERTS_DEPLOY_MINIO_CERT_PATH=/mnt/{{ iocage_pool_name }}/iocage/jails/minio_v2/root/home/minio/certs
+if [ "$3" == "True" ]; then
+    export CERTS_DEPLOY_POSTGRESQL_ENABLED=$3
+fi
+CERTS_DEPLOY_POSTGRESQL_PATH=/mnt/{{ postgresql_pool_name }}/postgresql

 # Check if cert is older than 69 days
 result=$(find ${CERTS_DEPLOY_PRIVATE_KEY_PATH} -mtime +69)
@@ -60,8 +66,29 @@ else
        set -o errexit
        echo "INFO - Certificate expires in less than $DAYS days"
        echo "INFO - Deploying new certificate"
-        # Deploy certificate (truenas UI & minio)
+        # Deploy certificate (truenas UI)
        python ${SCRIPT_PATH}/certificates_deploy.py
+        # Copy certificates (minio)
+        if [ "CERTS_DEPLOY_MINIO_ENABLED" == "True" ]; then
+            cp -pr ${CERTS_DEPLOY_PRIVATE_KEY_PATH} ${CERTS_DEPLOY_MINIO_CERT_PATH}/private.key
+            cp -pr ${CERTS_DEPLOY_FULLCHAIN_PATH} ${CERTS_DEPLOY_MINIO_CERT_PATH}/public.crt
+            iocage exec minio_v2 'service minio restart'
+        fi
+        # Copy certificates (postgresql)
+        if [ "CERTS_DEPLOY_POSTGRESQL_ENABLED" == "True" ]; then
+            pg_data_dirs=$(find /mnt/{{ postgresql_pool_name }}/postgresql -type d -maxdepth 1 -name '*data*' -exec basename {} \;)
+            for i in $pg_data_dirs; do
+                cp -pr ${CERTS_DEPLOY_PRIVATE_KEY_PATH} ${CERTS_DEPLOY_POSTGRESQL_PATH}/$i/server.key
+                cp -pr ${CERTS_DEPLOY_FULLCHAIN_PATH} ${CERTS_DEPLOY_POSTGRESQL_PATH}/$i/server.crt
+                iocage exec postgresql_v${i: -2} 'service postgresql reload'
+            done
+        fi
+        curl -s \
+            --form-string "token=${PUSHOVER_API_TOKEN}" \
+            --form-string "user=${PUSHOVER_USER_KEY}" \
+            --form-string "message=New Let's Encrypt certificate deployed on $TARGET." \
+            https://api.pushover.net/1/messages.json
+
    else
        echo "INFO - Certificate expires in more than $DAYS"
    fi
--- a/kubernetes/apps/default/homelab/truenas/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/truenas/kustomization.yaml
@@ -7,3 +7,4 @@ resources:
  - ./backup
  - ./certs-deploy
  - ./externalsecret.yaml
+  - ./pgdump
--- a/kubernetes/apps/default/homelab/truenas/pgdump/externalsecret.yaml
+++ b/kubernetes/apps/default/homelab/truenas/pgdump/externalsecret.yaml
@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: homelab-truenas-pgdump
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: homelab-truenas-pgdump-secret
+    template:
+      engineVersion: v2
+      data:
+        # App
+        POSTGRES_HOST: postgres.${SECRET_DOMAIN}
+        POSTGRES_USER: "{{ .POSTGRES_SUPER_USER }}"
+        POSTGRES_PASSWORD: "{{ .POSTGRES_SUPER_PASS }}"
+        POSTGRES_PORT: "5432"
+  dataFrom:
+
+    - extract:
+        key: generic
--- a/kubernetes/apps/default/homelab/truenas/pgdump/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/truenas/pgdump/helmrelease.yaml
@@ -0,0 +1,104 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: homelab-truenas-pgdump
+  namespace: default
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.4.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          concurrencyPolicy: Forbid
+          schedule: "@daily"
+        initContainers:
+          init-db:
+            image:
+              repository: ghcr.io/onedr0p/postgres-init
+              tag: 16
+            env:
+              EXCLUDE_DBS: "home_assistant radarr_log sonarr_log prowlarr_log postgres template0 template1"
+            envFrom: &envFrom
+              - secretRef:
+                  name: homelab-truenas-pgdump-secret
+            command:
+              - "/bin/bash"
+              - "-c"
+              - |
+                #!/bin/bash
+
+                set -o nounset
+                set -o errexit
+
+                # File to store the list of databases
+                OUTPUT_FILE="/config/db_list"
+
+                # Export PG password to avoid password prompt
+                export PGPASSWORD=$POSTGRES_PASSWORD
+
+                # Generate a regex pattern for exclusion
+                EXCLUDE_PATTERN=$(echo $EXCLUDE_DBS | sed 's/ /\\|/g')
+
+                # List all databases, exclude the ones in EXCLUDE_DBS, and write to the file
+                psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -lqt | \
+                cut -d \| -f 1 | \
+                grep -Ev "^\s*($EXCLUDE_PATTERN)\s*$" > "$OUTPUT_FILE"
+
+                # Unset PG password
+                unset PGPASSWORD
+
+                echo "Database list saved to $OUTPUT_FILE"
+
+                cat $OUTPUT_FILE
+        containers:
+          main:
+            image:
+              repository: prodrigestivill/postgres-backup-local
+              tag: 16-alpine
+            command: ["/backup.sh"]
+            env:
+              POSTGRES_DB_FILE: /config/db_list
+              POSTGRES_EXTRA_OPTS: "-Z9 --schema=public --blobs"
+              BACKUP_KEEP_DAYS: "7"
+              BACKUP_KEEP_WEEKS: "4"
+              BACKUP_KEEP_MONTHS: "3"
+              HEALTHCHECK_PORT: "8080"
+            envFrom: *envFrom
+    service:
+      main:
+        enabled: false
+    persistence:
+      config:
+        enabled: true
+        type: emptyDir
+        globalMounts:
+          - path: /config
+      backups:
+        enabled: true
+        type: nfs
+        server: "${LOCAL_LAN_TRUENAS}"
+        path: /mnt/storage/backups/postgresql
+        globalMounts:
+          - path: /backups
--- a/kubernetes/apps/default/homelab/truenas/pgdump/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/truenas/pgdump/kustomization.yaml
@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./externalsecret.yaml
+  - ./helmrelease.yaml