♻️ homelab

2025-09-17 18:24:14 +02:00 · 2023-11-12 20:45:54 +01:00
parent 886760adb7
commit 4ab17e0913
28 changed files with 183 additions and 98 deletions
--- a/kubernetes/apps/default/homelab/ks.yaml
+++ b/kubernetes/apps/default/homelab/ks.yaml
@@ -0,0 +1,60 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-homnelab-minio
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  path: ./kubernetes/apps/default/homelab/minio
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops-kubernetes
+  dependsOn:
+    - name: cluster-apps-external-secrets-stores
+  interval: 30m
+  retryInterval: 1m
+  timeout: 3m
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-homnelab-opnsense
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  path: ./kubernetes/apps/default/homelab/opnsense
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops-kubernetes
+  dependsOn:
+    - name: cluster-apps-external-secrets-stores
+  interval: 30m
+  retryInterval: 1m
+  timeout: 3m
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-homnelab-truenas
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  path: ./kubernetes/apps/default/homelab/truenas
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops-kubernetes
+  dependsOn:
+    - name: cluster-apps-external-secrets-stores
+  interval: 30m
+  retryInterval: 1m
+  timeout: 3m
--- a/kubernetes/apps/default/homelab/minio/backup/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/minio/backup/helmrelease.yaml
@@ -0,0 +1,63 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: homelab-minio-backup
+  namespace: default
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          concurrencyPolicy: Forbid
+          schedule: "@daily"
+        containers:
+          main:
+            image:
+              repository: ghcr.io/auricom/rclone
+              tag: 1.62.2@sha256:8d3ae01ed5295974be1b229f7398ce93a03c77a3fdaf301ea35bf929bb19389a
+            command: ["/bin/bash", "/app/minio-rclone.sh"]
+            envFrom:
+              - secretRef:
+                  name: homelab-minio-secret
+            service:
+              main:
+                enabled: false
+    service:
+      main:
+        enabled: false
+    persistence:
+      config:
+        enabled: true
+        type: configMap
+        name: homelab-minio-configmap
+        defaultMode: 0775
+        globalMounts:
+          - path: /app/minio-rclone.sh
+            subPath: minio-rclone.sh
+            readOnly: true
+          - path: /config/rclone.conf
+            subPath: rclone.conf
+            readOnly: true
--- a/kubernetes/apps/default/homelab/minio/backup/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/minio/backup/kustomization.yaml
@@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: homelab-minio-configmap
+    files:
+      - ./minio-rclone.sh
+      - ./rclone.conf
+generatorOptions:
+  disableNameSuffixHash: true
+
--- a/kubernetes/apps/default/homelab/minio/backup/minio-rclone.sh
+++ b/kubernetes/apps/default/homelab/minio/backup/minio-rclone.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+# Replace the placeholders in the file with the environment variables values
+cp /config/rclone.conf /tmp/rclone.conf
+sed -i "s@__RCLONE_ACCESS_ID__@$RCLONE_ACCESS_ID@g" "/tmp/rclone.conf"
+sed -i "s@__RCLONE_SECRET_KEY__@$RCLONE_SECRET_KEY@g" "/tmp/rclone.conf"
+sed -i "s@__PASSWORD__@$GDRIVE_PASSWORD@g" "/tmp/rclone.conf"
+sed -i "s@__PASSWORD2__@$GDRIVE_PASSWORD2@g" "/tmp/rclone.conf"
+sed -i "s@__GDRIVE_CLIENT_ID__@$GDRIVE_CLIENT_ID@g" "/tmp/rclone.conf"
+sed -i "s@__GDRIVE_CLIENT_SECRET__@$GDRIVE_CLIENT_SECRET@g" "/tmp/rclone.conf"
+sed -i "s@__GDRIVE_TOKEN__@$GDRIVE_TOKEN@g" "/tmp/rclone.conf"
+
+echo "Sync minio buckets with encrypted remote gdrive-homelab-backups ..."
+rclone --config /tmp/rclone.conf sync minio: gdrive-homelab-backups:
--- a/kubernetes/apps/default/homelab/minio/backup/rclone.conf
+++ b/kubernetes/apps/default/homelab/minio/backup/rclone.conf
@@ -0,0 +1,22 @@
+[minio]
+type = s3
+provider = Minio
+access_key_id = __RCLONE_ACCESS_ID__
+secret_access_key = __RCLONE_SECRET_KEY__
+endpoint = https://minio.${SECRET_DOMAIN}:51515
+acl = private
+
+[gdrive-homelab-backups]
+type = crypt
+remote = gdrive:homelab-backups
+directory_name_encryption = false
+password = __PASSWORD__
+password2 = __PASSWORD2__
+
+[gdrive]
+type = drive
+client_id = __GDRIVE_CLIENT_ID__
+client_secret = __GDRIVE_CLIENT_SECRET__
+scope = drive.file
+token = __GDRIVE_TOKEN__
+team_drive =
--- a/kubernetes/apps/default/homelab/minio/externalsecret.yaml
+++ b/kubernetes/apps/default/homelab/minio/externalsecret.yaml
@@ -0,0 +1,28 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: homelab-minio
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: homelab-minio-secret
+    creationPolicy: Owner
+    template:
+      data:
+        # App
+        GDRIVE_CLIENT_ID: "{{ .GDRIVE_CLIENT_ID }}"
+        GDRIVE_CLIENT_SECRET: "{{ .GDRIVE_CLIENT_SECRET }}"
+        GDRIVE_TOKEN: "{{ .GDRIVE_TOKEN }}"
+        GDRIVE_PASSWORD: "{{ .GDRIVE_PASSWORD }}"
+        GDRIVE_PASSWORD2: "{{ .GDRIVE_PASSWORD2 }}"
+        RCLONE_ACCESS_ID: "{{ .RCLONE_ACCESS_ID }}"
+        RCLONE_SECRET_KEY: "{{ .RCLONE_SECRET_KEY }}"
+
+  dataFrom:
+    - extract:
+        key: homelab-minio
--- a/kubernetes/apps/default/homelab/minio/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/minio/kustomization.yaml
@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./backup
+  - ./externalsecret.yaml
--- a/kubernetes/apps/default/homelab/opnsense/backup/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/opnsense/backup/helmrelease.yaml
@@ -0,0 +1,60 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: homelab-opnsense-backup
+  namespace: default
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          concurrencyPolicy: Forbid
+          schedule: "@daily"
+        containers:
+          main:
+            image:
+              repository: ghcr.io/auricom/kubectl
+              tag: 1.28.3@sha256:536e3a2a8222d56637208c207a5b77a7d656175a29b899383d5a1bb1d1e48438
+            command: ["/bin/bash", "/app/opnsense-backup.sh"]
+            env:
+              OPNSENSE_URL: "https://opnsense.${SECRET_DOMAIN}"
+              S3_URL: "https://truenas.${SECRET_DOMAIN}:51515"
+            envFrom:
+              - secretRef:
+                  name: homelab-opnsense-secret
+    service:
+      main:
+        enabled: false
+    persistence:
+      config:
+        enabled: true
+        type: configMap
+        name: homelab-opnsense-backup-configmap
+        defaultMode: 0775
+        globalMounts:
+          - path: /app/opnsense-backup.sh
+            subPath: opnsense-backup.sh
+            readOnly: true
--- a/kubernetes/apps/default/homelab/opnsense/backup/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/opnsense/backup/kustomization.yaml
@@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: homelab-opnsense-backup-configmap
+    files:
+      - ./opnsense-backup.sh
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled
--- a/kubernetes/apps/default/homelab/opnsense/backup/opnsense-backup.sh
+++ b/kubernetes/apps/default/homelab/opnsense/backup/opnsense-backup.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+config_filename="$(date "+%Y%m%d-%H%M%S").xml"
+
+http_host=${S3_URL#*//}
+http_host=${http_host%:*}
+http_request_date=$(date -R)
+http_filepath="opnsense/${config_filename}"
+http_signature=$(
+    printf "PUT\n\ntext/xml\n%s\n/%s" "${http_request_date}" "${http_filepath}" \
+        | openssl sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary \
+        | base64
+)
+
+echo "Download Opnsense config file ..."
+curl -fsSL \
+    --user "${OPNSENSE_KEY}:${OPNSENSE_SECRET}" \
+    --output "/tmp/${config_filename}" \
+    "${OPNSENSE_URL}/api/backup/backup/download"
+
+echo "Upload backup to s3 bucket ..."
+curl -fsSL \
+    -X PUT -T "/tmp/${config_filename}" \
+    -H "Host: ${http_host}" \
+    -H "Date: ${http_request_date}" \
+    -H "Content-Type: text/xml" \
+    -H "Authorization: AWS ${AWS_ACCESS_KEY_ID}:${http_signature}" \
+    "${S3_URL}/${http_filepath}"
--- a/kubernetes/apps/default/homelab/opnsense/backup/readme.md
+++ b/kubernetes/apps/default/homelab/opnsense/backup/readme.md
@@ -0,0 +1,71 @@
+# Opnsense
+
+## S3 Configuration
+
+1. Create `~/.mc/config.json`
+
+   ```json
+   {
+     "version": "10",
+     "aliases": {
+       "minio": {
+         "url": "https://s3.<domain>",
+         "accessKey": "<access-key>",
+         "secretKey": "<secret-key>",
+         "api": "S3v4",
+         "path": "auto"
+       }
+     }
+   }
+   ```
+
+2. Create the opnsense user and password
+
+   ```sh
+   mc admin user add minio opnsense <super-secret-password>
+   ```
+
+3. Create the opnsense bucket
+
+   ```sh
+   mc mb minio/opnsense
+   ```
+
+4. Create `opnsense-user-policy.json`
+
+   ```json
+   {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Action": [
+           "s3:ListBucket",
+           "s3:PutObject",
+           "s3:GetObject",
+           "s3:DeleteObject"
+         ],
+         "Effect": "Allow",
+         "Resource": ["arn:aws:s3:::opnsense/*", "arn:aws:s3:::opnsense"],
+         "Sid": ""
+       }
+     ]
+   }
+   ```
+
+5. Apply the bucket policies
+
+    ```sh
+    mc admin policy add minio opnsense-private opnsense-user-policy.json
+    ```
+
+6. Associate private policy with the user
+
+    ```sh
+    mc admin policy set minio opnsense-private user=opnsense
+    ```
+
+7. Create a retention policy
+
+    ```sh
+    mc ilm add minio/opnsense --expire-days "90"
+    ```
--- a/kubernetes/apps/default/homelab/opnsense/dashboard.json
+++ b/kubernetes/apps/default/homelab/opnsense/dashboard.json
@@ -0,0 +1,202 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 66,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "circleMaxSize": 30,
+      "circleMinSize": 2,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "decimals": 0,
+      "esMetric": "Count",
+      "gridPos": {
+        "h": 11,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hideEmpty": false,
+      "hideZero": false,
+      "id": 2,
+      "initialZoom": 1,
+      "locationData": "countries",
+      "mapCenter": "(0°, 0°)",
+      "mapCenterLatitude": 0,
+      "mapCenterLongitude": 0,
+      "maxDataPoints": 1,
+      "mouseWheelZoom": false,
+      "showLegend": true,
+      "stickyLabels": false,
+      "tableQueryOptions": {
+        "geohashField": "geohash",
+        "latitudeField": "latitude",
+        "longitudeField": "longitude",
+        "metricField": "metric",
+        "queryType": "geohash"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "expr": "sum(count_over_time({hostname=\"opnsense\"} | json | appname = \"filterlog\" | filter_action = \"pass\"[$__interval])) by (geoip_country_code)",
+          "legendFormat": "{{geoip_country_code}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "0,10",
+      "title": "Allowed incoming connections by GeoIP",
+      "type": "grafana-worldmap-panel",
+      "unitPlural": "",
+      "unitSingle": "",
+      "valueName": "total"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "displayMode": "lcd",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true
+      },
+      "pluginVersion": "8.3.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "expr": "topk(10, \n  sum by (filter_destination_port) (\n  count_over_time(\n      {hostname=\"opnsense\"} \n      | json \n      | appname = \"filterlog\"\n      | filter_destination_port != \"\"\n      | filter_action = \"pass\"\n      | filter_interface = \"igb0\"\n      [$__range]\n    )\n  )\n)",
+          "instant": true,
+          "legendFormat": "{{filter_destination_port}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 allowed incoming ports",
+      "transformations": [
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "Value #A"
+              }
+            ]
+          }
+        },
+        {
+          "id": "rowsToFields",
+          "options": {
+            "mappings": [
+              {
+                "fieldName": "Time",
+                "handlerKey": "field.value"
+              },
+              {
+                "fieldName": "filter_source_port",
+                "handlerKey": "field.name"
+              },
+              {
+                "fieldName": "Value #A",
+                "handlerKey": "field.value"
+              },
+              {
+                "fieldName": "filter_destination_port",
+                "handlerKey": "field.name"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "bargauge"
+    }
+  ],
+  "schemaVersion": 34,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "OPNsense",
+  "uid": "itdu1LAnk",
+  "version": 9,
+  "weekStart": ""
+}
--- a/kubernetes/apps/default/homelab/opnsense/externalsecret.yaml
+++ b/kubernetes/apps/default/homelab/opnsense/externalsecret.yaml
@@ -0,0 +1,18 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: homelab-opnsense
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: homelab-opnsense-secret
+    creationPolicy: Owner
+  dataFrom:
+    - extract:
+        # OPNSENSE_KEY, OPNSENSE_SECRET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
+        key: homelab-opnsense
--- a/kubernetes/apps/default/homelab/opnsense/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/opnsense/kustomization.yaml
@@ -0,0 +1,18 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./backup
+  - ./externalsecret.yaml
+configMapGenerator:
+  - name: opnsense-dashboard
+    files:
+      - opnsense-dashboard.json=./dashboard.json
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled
+  labels:
+    grafana_dashboard: "true"
--- a/kubernetes/apps/default/homelab/truenas/backup/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/truenas/backup/helmrelease.yaml
@@ -0,0 +1,77 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: homelab-truenas-backup
+  namespace: default
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          concurrencyPolicy: Forbid
+          schedule: "@daily"
+        containers:
+          main:
+            image:
+              repository: ghcr.io/auricom/kubectl
+              tag: 1.28.3@sha256:536e3a2a8222d56637208c207a5b77a7d656175a29b899383d5a1bb1d1e48438
+            command: ["/bin/bash", "/app/truenas-backup.sh"]
+            env:
+              HOSTNAME: truenas
+            envFrom: &envFrom
+              - secretRef:
+                  name: &secret homelab-truenas-secret
+          truenas-remote-backup:
+            name: truenas-remote-backup
+            image:
+              repository: ghcr.io/auricom/kubectl
+              tag: 1.28.3@sha256:536e3a2a8222d56637208c207a5b77a7d656175a29b899383d5a1bb1d1e48438
+            command: ["/bin/bash", "/app/truenas-backup.sh"]
+            env:
+              HOSTNAME: truenas-remote
+            envFrom: *envFrom
+    service:
+      main:
+        enabled: false
+    persistence:
+      config:
+        enabled: true
+        type: configMap
+        name: homelab-truenas-backup-configmap
+        defaultMode: 0775
+        globalMounts:
+          - path: /app/truenas-backup.sh
+            subPath: truenas-backup.sh
+            readOnly: true
+      ssh:
+        type: secret
+        name: *secret
+        defaultMode: 0775
+        globalMounts:
+          - path: /opt/id_rsa
+            subPath: TRUENAS_SSH_KEY
+            readOnly: true
+
--- a/kubernetes/apps/default/homelab/truenas/backup/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/truenas/backup/kustomization.yaml
@@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: homelab-truenas-backup-configmap
+    files:
+      - ./truenas-backup.sh
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled
--- a/kubernetes/apps/default/homelab/truenas/backup/truenas-backup.sh
+++ b/kubernetes/apps/default/homelab/truenas/backup/truenas-backup.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+mkdir -p ~/.ssh
+cp /opt/id_rsa ~/.ssh/id_rsa
+chmod 600 ~/.ssh/id_rsa
+
+printf -v aws_access_key_id_str %q "$TRUENAS_AWS_ACCESS_KEY_ID"
+printf -v aws_secret_access_key_str %q "$TRUENAS_AWS_SECRET_ACCESS_KEY"
+printf -v secret_domain_str %q "$SECRET_DOMAIN"
+
+
+ssh -o StrictHostKeyChecking=no root@${HOSTNAME}.${SECRET_DOMAIN} "/bin/bash -s $aws_access_key_id_str $aws_secret_access_key_str $secret_domain_str" << 'EOF'
+
+set -o nounset
+set -o errexit
+
+AWS_ACCESS_KEY_ID=$1
+AWS_SECRET_ACCESS_KEY=$2
+SECRET_DOMAIN=$3
+
+config_filename="$(date "+%Y%m%d-%H%M%S").tar"
+
+http_host=truenas.${SECRET_DOMAIN}
+http_request_date=$(date -R)
+http_content_type="application/x-tar"
+http_filepath="truenas/$(hostname)/${config_filename}"
+http_signature=$(
+    printf "PUT\n\n${http_content_type}\n%s\n/%s" "${http_request_date}" "${http_filepath}" \
+        | openssl sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary \
+        | base64
+)
+
+echo "Creating backup archive ..."
+
+tar -cvlf /tmp/backup-${config_filename} --strip-components=2 /data/freenas-v1.db /data/pwenc_secret
+
+echo "Upload backup to s3 bucket ..."
+curl -fsSL \
+    -X PUT -T "/tmp/backup-${config_filename}" \
+    -H "Host: ${http_host}" \
+    -H "Date: ${http_request_date}" \
+    -H "Content-Type: ${http_content_type}" \
+    -H "Authorization: AWS ${AWS_ACCESS_KEY_ID}:${http_signature}" \
+    "https://truenas.${SECRET_DOMAIN}:51515/${http_filepath}"
+
+rm /tmp/backup-*.tar
+
+EOF
--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/helmrelease.yaml
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/helmrelease.yaml
@@ -0,0 +1,88 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: homelab-truenas-certs-deploy
+  namespace: default
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          concurrencyPolicy: Forbid
+          schedule: "@daily"
+        containers:
+          main:
+            image:
+              repository: ghcr.io/auricom/kubectl
+              tag: 1.28.3@sha256:536e3a2a8222d56637208c207a5b77a7d656175a29b899383d5a1bb1d1e48438
+            command: ["/bin/bash", "/app/truenas-certs-deploy.sh"]
+            env:
+              HOSTNAME: truenas
+              TRUENAS_HOME: /mnt/storage/home/homelab
+              CERTS_DEPLOY_S3_ENABLED: "True"
+            envFrom: &envFrom
+              - secretRef:
+                  name: &secret homelab-truenas-secret
+          truenas-remote-certs-deploy:
+            image:
+              repository: ghcr.io/auricom/kubectl
+              tag: 1.28.3@sha256:536e3a2a8222d56637208c207a5b77a7d656175a29b899383d5a1bb1d1e48438
+            command: ["/bin/bash", "/app/truenas-certs-deploy.sh"]
+            env:
+              HOSTNAME: truenas-remote
+              TRUENAS_HOME: /mnt/vol1/home/homelab
+              CERTS_DEPLOY_S3_ENABLED: "False"
+            envFrom: *envFrom
+    service:
+      main:
+        enabled: false
+    persistence:
+      config:
+        enabled: true
+        type: configMap
+        name: homelab-truenas-certs-deploy-configmap
+        defaultMode: 0775
+        globalMounts:
+          - path: /app/truenas-certs-deploy.sh
+            subPath: truenas-certs-deploy.sh
+            readOnly: true
+      config-python:
+        type: configMap
+        name: homelab-truenas-certs-deploy-configmap
+        defaultMode: 0775
+        globalMounts:
+          - path: /app/truenas-certs-deploy.py
+            subPath: truenas-certs-deploy.py
+            readOnly: true
+      ssh:
+        type: secret
+        name: *secret
+        defaultMode: 0775
+        globalMounts:
+          - path: /opt/id_rsa
+            subPath: TRUENAS_SSH_KEY
+            readOnly: true
+
--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/kustomization.yaml
@@ -0,0 +1,16 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: homelab-truenas-certs-deploy-configmap
+    files:
+      - ./truenas-certs-deploy.sh
+      - ./truenas-certs-deploy.py
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled
--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.py
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+
+"""
+Import and activate a SSL/TLS certificate into FreeNAS 11.1 or later
+Uses the FreeNAS API to make the change, so everything's properly saved in the config
+database and captured in a backup.
+
+Requires paths to the cert (including the any intermediate CA certs) and private key,
+and username, password, and FQDN of your FreeNAS system.
+
+Source: https://github.com/danb35/deploy-freenas
+"""
+
+import argparse
+import os
+import sys
+import json
+import requests
+import time
+import configparser
+import socket
+from datetime import datetime, timedelta
+from urllib3.exceptions import InsecureRequestWarning
+requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+
+API_KEY = os.getenv('CERTS_DEPLOY_API_KEY')
+
+DOMAIN_NAME = socket.gethostname()
+TRUENAS_ADDRESS = 'localhost'
+VERIFY = False
+PRIVATEKEY_PATH = os.getenv('CERTS_DEPLOY_PRIVATE_KEY_PATH')
+FULLCHAIN_PATH = os.getenv('CERTS_DEPLOY_FULLCHAIN_PATH')
+PROTOCOL = 'http://'
+PORT = '80'
+FTP_ENABLED = bool(os.getenv('CERTS_DEPLOY_FTP_ENABLED', ''))
+S3_ENABLED = bool(os.getenv('CERTS_DEPLOY_S3_ENABLED', ''))
+now = datetime.now()
+cert = "letsencrypt-%s-%s-%s-%s" %(now.year, now.strftime('%m'), now.strftime('%d'), ''.join(c for c in now.strftime('%X') if
+c.isdigit()))
+
+
+# Set some general request params
+session = requests.Session()
+session.headers.update({
+  'Content-Type': 'application/json'
+})
+if API_KEY:
+  session.headers.update({
+    'Authorization': f'Bearer {API_KEY}'
+  })
+else:
+  print ("Unable to authenticate. Specify 'CERTS_DEPLOY_API_KEY' in the os Env.")
+  exit(1)
+if not PRIVATEKEY_PATH:
+  print ("Unable to find private key. Specify 'CERTS_DEPLOY_PRIVATE_KEY_PATH' in the os Env.")
+  exit(1)
+if not FULLCHAIN_PATH:
+  print ("Unable to find private key. Specify 'CERTS_DEPLOY_FULLCHAIN_PATH' in the os Env.")
+  exit(1)
+
+# Load cert/key
+with open(PRIVATEKEY_PATH, 'r') as file:
+  priv_key = file.read()
+with open(FULLCHAIN_PATH, 'r') as file:
+  full_chain = file.read()
+
+# Update or create certificate
+r = session.post(
+  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/',
+  verify=VERIFY,
+  data=json.dumps({
+    "create_type": "CERTIFICATE_CREATE_IMPORTED",
+    "name": cert,
+    "certificate": full_chain,
+    "privatekey": priv_key,
+  })
+)
+
+if r.status_code == 200:
+  print ("Certificate import successful")
+else:
+  print ("Error importing certificate!")
+  print (r.text)
+  sys.exit(1)
+
+# Sleep for a few seconds to let the cert propagate
+time.sleep(5)
+
+# Download certificate list
+limit = {'limit': 0} # set limit to 0 to disable paging in the event of many certificates
+r = session.get(
+  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/',
+  verify=VERIFY,
+  params=limit
+)
+
+if r.status_code == 200:
+  print ("Certificate list successful")
+else:
+  print ("Error listing certificates!")
+  print (r.text)
+  sys.exit(1)
+
+# Parse certificate list to find the id that matches our cert name
+cert_list = r.json()
+
+new_cert_data = None
+for cert_data in cert_list:
+  if cert_data['name'] == cert:
+    new_cert_data = cert_data
+    cert_id = new_cert_data['id']
+    break
+
+if not new_cert_data:
+  print ("Error searching for newly imported certificate in certificate list.")
+  sys.exit(1)
+
+# Set our cert as active
+r = session.put(
+  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/system/general/',
+  verify=VERIFY,
+  data=json.dumps({
+    "ui_certificate": cert_id,
+  })
+)
+
+if r.status_code == 200:
+  print ("Setting active certificate successful")
+else:
+  print ("Error setting active certificate!")
+  print (r.text)
+  sys.exit(1)
+
+if FTP_ENABLED:
+  # Set our cert as active for FTP plugin
+  r = session.put(
+    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/ftp/',
+    verify=VERIFY,
+    data=json.dumps({
+      "ssltls_certfile": cert,
+    }),
+  )
+
+  if r.status_code == 200:
+    print ("Setting active FTP certificate successful")
+  else:
+    print ("Error setting active FTP certificate!")
+    print (r.text)
+    sys.exit(1)
+
+if S3_ENABLED:
+  # Set our cert as active for S3 plugin
+  r = session.put(
+    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/s3/',
+    verify=VERIFY,
+    data=json.dumps({
+      "certificate": cert_id,
+    }),
+  )
+
+  if r.status_code == 200:
+    print ("Setting active S3 certificate successful")
+  else:
+    print ("Error setting active S3 certificate!")
+    print (r)
+    sys.exit(1)
+
+# Get expired and old certs with same SAN
+cert_ids_same_san = set()
+cert_ids_expired = set()
+for cert_data in cert_list:
+  if set(cert_data['san']) == set(new_cert_data['san']):
+      cert_ids_same_san.add(cert_data['id'])
+
+  issued_date = datetime.strptime(cert_data['from'], "%c")
+  lifetime = timedelta(days=cert_data['lifetime'])
+  expiration_date = issued_date + lifetime
+  if expiration_date < now:
+      cert_ids_expired.add(cert_data['id'])
+
+# Remove new cert_id from lists
+if cert_id in cert_ids_expired:
+  cert_ids_expired.remove(cert_id)
+
+if cert_id in cert_ids_same_san:
+  cert_ids_same_san.remove(cert_id)
+
+# Delete expired and old certificates with same SAN from freenas
+for cid in (cert_ids_same_san | cert_ids_expired):
+  r = session.delete(
+    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/id/' + str(cid),
+    verify=VERIFY
+  )
+
+  for c in cert_list:
+    if c['id'] == cid:
+      cert_name = c['name']
+
+  if r.status_code == 200:
+    print ("Deleting certificate " + cert_name + " successful")
+  else:
+    print ("Error deleting certificate " + cert_name + "!")
+    print (r.text)
+    sys.exit(1)
+
+# Reload nginx with new cert
+# If everything goes right, the request fails with a ConnectionError
+try:
+  r = session.post(
+    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/system/general/ui_restart',
+    verify=VERIFY
+  )
+
+  if r.status_code == 200:
+    print ("Reloading WebUI successful")
+    print ("deploy_freenas.py executed successfully")
+  else:
+    print ("Error reloading WebUI!")
+    print ("{}: {}".format(r.status_code, r.text))
+    sys.exit(1)
+except requests.exceptions.ConnectionError:
+    print ("Error reloading WebUI!")
+    sys.exit(1)
--- a/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.sh
+++ b/kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+mkdir -p ~/.ssh
+cp /opt/id_rsa ~/.ssh/id_rsa
+chmod 600 ~/.ssh/id_rsa
+
+if [ "${HOSTNAME}" == "truenas" ]; then
+    printf -v truenas_api_key %q "$TRUENAS_API_KEY"
+elif [ "${HOSTNAME}" == "truenas-remote" ]; then
+    printf -v truenas_api_key %q "$TRUENAS_REMOTE_API_KEY"
+fi
+printf -v cert_deploy_s3_enabled_str %q "$CERTS_DEPLOY_S3_ENABLED"
+printf -v pushover_api_token_str %q "$PUSHOVER_API_TOKEN"
+printf -v pushover_user_key_str %q "$PUSHOVER_USER_KEY"
+printf -v secret_domain_str %q "$SECRET_DOMAIN"
+
+scp -o StrictHostKeyChecking=no /app/truenas-certs-deploy.py homelab@${HOSTNAME}.${SECRET_DOMAIN}:${TRUENAS_HOME}/scripts/certificates_deploy.py
+
+ssh -o StrictHostKeyChecking=no homelab@${HOSTNAME}.${SECRET_DOMAIN} "/bin/bash -s $truenas_api_key $cert_deploy_s3_enabled_str $pushover_api_token_str $pushover_user_key_str $secret_domain_str" << 'EOF'
+
+set -o nounset
+set -o errexit
+
+PUSHOVER_API_TOKEN=$3
+PUSHOVER_USER_KEY=$4
+SECRET_DOMAIN=$5
+
+# Variables
+TARGET=$(hostname)
+DAYS=21
+CERTIFICATE_PATH="${HOME}/letsencrypt/${SECRET_DOMAIN}"
+SCRIPT_PATH="${HOME}/scripts"
+
+export CERTS_DEPLOY_API_KEY=$1
+export CERTS_DEPLOY_PRIVATE_KEY_PATH=${CERTIFICATE_PATH}/key.pem
+export CERTS_DEPLOY_FULLCHAIN_PATH=${CERTIFICATE_PATH}/fullchain.pem
+if [ "$2" == "True" ]; then
+    export CERTS_DEPLOY_S3_ENABLED=$2
+fi
+
+# Check if cert is older than 69 days
+result=$(find ${CERTS_DEPLOY_PRIVATE_KEY_PATH} -mtime +69)
+
+if [[ "$result" == "${CERTS_DEPLOY_PRIVATE_KEY_PATH}" ]]; then
+    echo "ERROR - Certificate is older than 69 days"
+    echo "ERROR - Verify than it has been renewed by ACME client on opnsense and that the upload automation has been executed"
+    curl -s \
+        --form-string "token=${PUSHOVER_API_TOKEN}" \
+        --form-string "user=${PUSHOVER_USER_KEY}" \
+        --form-string "message=Certificate on $TARGET is older than 69 days. Verify than it has been renewed by ACME client on opnsense and that the upload automation has been executed" \
+        https://api.pushover.net/1/messages.json
+else
+    echo "INFO checking if $TARGET expires in less than $DAYS days"
+    set +o errexit
+    openssl x509 -checkend $(( 24*3600*$DAYS )) -noout -in <(openssl s_client -showcerts -connect $TARGET:443 </dev/null 2>/dev/null | openssl x509 -outform PEM)
+    if [[ $? -ne 0 ]]; then
+        set -o errexit
+        echo "INFO - Certificate expires in less than $DAYS days"
+        echo "INFO - Deploying new certificate"
+        # Deploy certificate (truenas UI & minio)
+        python ${SCRIPT_PATH}/certificates_deploy.py
+    else
+        echo "INFO - Certificate expires in more than $DAYS"
+    fi
+fi
+
+EOF
--- a/kubernetes/apps/default/homelab/truenas/externalsecret.yaml
+++ b/kubernetes/apps/default/homelab/truenas/externalsecret.yaml
@@ -0,0 +1,35 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: homelab-truenas
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: homelab-truenas-secret
+    creationPolicy: Owner
+    template:
+      data:
+        # App
+        PUSHOVER_API_TOKEN: "{{ .TRUENAS_PUSHOVER_API_TOKEN }}"
+        PUSHOVER_USER_KEY: "{{ .PUSHOVER_USER_KEY }}"
+        TRUENAS_AWS_ACCESS_KEY_ID: "{{ .TRUENAS_AWS_ACCESS_KEY_ID }}"
+        TRUENAS_AWS_SECRET_ACCESS_KEY: "{{ .TRUENAS_AWS_SECRET_ACCESS_KEY }}"
+        TRUENAS_SSH_KEY: "{{ .TRUENAS_SSH_KEY }}"
+        TRUENAS_API_KEY: "{{ .TRUENAS_API_KEY }}"
+        TRUENAS_REMOTE_API_KEY: "{{ .TRUENAS_REMOTE_API_KEY }}"
+        SECRET_DOMAIN: "{{ .SECRET_DOMAIN }}"
+        SECRET_PUBLIC_DOMAIN: "{{ .SECRET_PUBLIC_DOMAIN }}"
+  dataFrom:
+    - extract:
+        key: generic
+    - extract:
+        key: homelab-truenas
+    - extract:
+        key: pushover
+    - extract:
+        key: sops
--- a/kubernetes/apps/default/homelab/truenas/kustomization.yaml
+++ b/kubernetes/apps/default/homelab/truenas/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ./backup
+  - ./certs-deploy
+  - ./externalsecret.yaml
--- a/kubernetes/apps/default/homelab/truenas/readme.md
+++ b/kubernetes/apps/default/homelab/truenas/readme.md
@@ -0,0 +1,128 @@
+# truenas
+
+## truenas-backup S3 Configuration
+
+1. Create `~/.mc/config.json`
+
+   ```json
+   {
+     "version": "10",
+     "aliases": {
+       "minio": {
+         "url": "https://s3.<domain>",
+         "accessKey": "<access-key>",
+         "secretKey": "<secret-key>",
+         "api": "S3v4",
+         "path": "auto"
+       }
+     }
+   }
+   ```
+
+2. Create the truenas user and password
+
+   ```sh
+   mc admin user add minio truenas <super-secret-password>
+   ```
+
+3. Create the truenas bucket
+
+   ```sh
+   mc mb minio/truenas
+   ```
+
+4. Create `truenas-user-policy.json`
+
+   ```json
+   {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Action": [
+           "s3:ListBucket",
+           "s3:PutObject",
+           "s3:GetObject",
+           "s3:DeleteObject"
+         ],
+         "Effect": "Allow",
+         "Resource": ["arn:aws:s3:::truenas/*", "arn:aws:s3:::truenas"],
+         "Sid": ""
+       }
+     ]
+   }
+   ```
+
+5. Apply the bucket policies
+
+    ```sh
+    mc admin policy add minio truenas-private truenas-user-policy.json
+    ```
+
+6. Associate private policy with the user
+
+    ```sh
+    mc admin policy set minio truenas-private user=truenas
+    ```
+
+7. Create a retention policy
+
+    ```sh
+    mc ilm add minio/truenas --expire-days "90"
+    ```
+
+## minio-rclone S3 Configuration
+
+1. Create `~/.mc/config.json`
+
+   ```json
+   {
+     "version": "10",
+     "aliases": {
+       "minio": {
+         "url": "https://s3.<domain>",
+         "accessKey": "<access-key>",
+         "secretKey": "<secret-key>",
+         "api": "S3v4",
+         "path": "auto"
+       }
+     }
+   }
+   ```
+
+2. Create the rclone user and password
+
+   ```sh
+   mc admin user add minio rclone <super-secret-password>
+   ```
+
+
+3. Create `rclone-user-policy.json`
+
+   ```json
+   {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Action": [
+           "s3:ListBucket",
+           "s3:GetObject"
+         ],
+         "Effect": "Allow",
+         "Resource": ["arn:aws:s3:::opnsense/*", "arn:aws:s3:::opnsense","arn:aws:s3:::truenas/*", "arn:aws:s3:::truenas"],
+         "Sid": ""
+       }
+     ]
+   }
+   ```
+
+4. Apply the bucket policies
+
+    ```sh
+    mc admin policy add minio rclone-private rclone-user-policy.json
+    ```
+
+5. Associate private policy with the user
+
+    ```sh
+    mc admin policy set minio rclone-private user=rclone
+    ```