feat: nas refactor

kubernetes/apps/default/homelab/minio/backup/rclone.conf

@@ -3,7 +3,7 @@ type = s3
 provider = Minio
 access_key_id = __RCLONE_ACCESS_ID__
 secret_access_key = __RCLONE_SECRET_KEY__
-endpoint = https://minio.${SECRET_DOMAIN}:9000
+endpoint = https://s3.feisar.ovh
 acl = private
 
 [gdrive-homelab-backups]

kubernetes/apps/default/homelab/opnsense/backup/helmrelease.yaml

@@ -41,8 +41,8 @@ spec:
               tag: 1.29.2@sha256:693ced2697bb7c7349419d4035a62bd474fc41710675b344f71773d8a687dfc3
             command: [/bin/bash, /app/opnsense-backup.sh]
             env:
-              OPNSENSE_URL: "https://opnsense.${SECRET_DOMAIN}"
-              S3_URL: "https://minio.${SECRET_DOMAIN}:9000"
+              OPNSENSE_URL: "https://opnsense.feisar.ovh"
+              S3_URL: "https://s3.feisar.ovh"
             envFrom:
               - secretRef:
                   name: homelab-opnsense-secret

kubernetes/apps/default/homelab/truenas/backup/truenas-backup.sh

@@ -44,7 +44,7 @@ curl -fsSL \
     -H "Date: ${http_request_date}" \
     -H "Content-Type: ${http_content_type}" \
     -H "Authorization: AWS ${AWS_ACCESS_KEY_ID}:${http_signature}" \
-    "https://minio.${SECRET_DOMAIN}:9000/${http_filepath}"
+    "https://s3.feisar.ovh/${http_filepath}"
 
 rm /tmp/backup-*.tar
 

kubernetes/apps/default/homelab/truenas/certs-deploy/helmrelease.yaml

@@ -1,91 +0,0 @@
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
-kind: HelmRelease
-metadata:
-  name: &app homelab-truenas-certs-deploy
-  namespace: default
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: app-template
-      version: 3.1.0
-      sourceRef:
-        kind: HelmRepository
-        name: bjw-s
-        namespace: flux-system
-  maxHistory: 2
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      strategy: rollback
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    controllers:
-      homelab-truenas-certs-deploy:
-        type: cronjob
-        cronjob:
-          concurrencyPolicy: Forbid
-          schedule: "@daily"
-        containers:
-          app:
-            image:
-              repository: ghcr.io/auricom/kubectl
-              tag: 1.29.2@sha256:693ced2697bb7c7349419d4035a62bd474fc41710675b344f71773d8a687dfc3
-            command: [/bin/bash, /app/truenas-certs-deploy.sh]
-            env:
-              HOSTNAME: truenas
-              TRUENAS_HOME: /mnt/storage/home/homelab
-              CERTS_DEPLOY_MINIO_ENABLED: "True"
-              CERTS_DEPLOY_POSTGRESQL_ENABLED: "True"
-            envFrom: &envFrom
-              - secretRef:
-                  name: &secret homelab-truenas-secret
-          truenas-remote-certs-deploy:
-            image:
-              repository: ghcr.io/auricom/kubectl
-              tag: 1.29.2@sha256:693ced2697bb7c7349419d4035a62bd474fc41710675b344f71773d8a687dfc3
-            command: [/bin/bash, /app/truenas-certs-deploy.sh]
-            env:
-              HOSTNAME: truenas-remote
-              TRUENAS_HOME: /mnt/vol1/home/homelab
-              CERTS_DEPLOY_MINIO_ENABLED: "False"
-              CERTS_DEPLOY_POSTGRESQL_ENABLED: "False"
-            envFrom: *envFrom
-    service:
-      app:
-        controller: *app
-        enabled: false
-    persistence:
-      config:
-        enabled: true
-        type: configMap
-        name: homelab-truenas-certs-deploy-configmap
-        defaultMode: 0775
-        globalMounts:
-          - path: /app/truenas-certs-deploy.sh
-            subPath: truenas-certs-deploy.sh
-            readOnly: true
-      config-python:
-        type: configMap
-        name: homelab-truenas-certs-deploy-configmap
-        defaultMode: 0775
-        globalMounts:
-          - path: /app/truenas-certs-deploy.py
-            subPath: truenas-certs-deploy.py
-            readOnly: true
-      ssh:
-        type: secret
-        name: *secret
-        defaultMode: 0775
-        globalMounts:
-          - path: /opt/id_rsa
-            subPath: TRUENAS_SSH_KEY
-            readOnly: true

kubernetes/apps/default/homelab/truenas/certs-deploy/kustomization.yaml

@@ -1,16 +0,0 @@
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: default
-resources:
-  - ./helmrelease.yaml
-configMapGenerator:
-  - name: homelab-truenas-certs-deploy-configmap
-    files:
-      - ./truenas-certs-deploy.sh
-      - ./truenas-certs-deploy.py
-generatorOptions:
-  disableNameSuffixHash: true
-  annotations:
-    kustomize.toolkit.fluxcd.io/substitute: disabled

kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.py

@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Import and activate a SSL/TLS certificate into FreeNAS 11.1 or later
-Uses the FreeNAS API to make the change, so everything's properly saved in the config
-database and captured in a backup.
-
-Requires paths to the cert (including the any intermediate CA certs) and private key,
-and username, password, and FQDN of your FreeNAS system.
-
-Source: https://github.com/danb35/deploy-freenas
-"""
-
-import argparse
-import os
-import sys
-import json
-import requests
-import time
-import configparser
-import socket
-from datetime import datetime, timedelta
-from urllib3.exceptions import InsecureRequestWarning
-requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
-
-API_KEY = os.getenv('CERTS_DEPLOY_API_KEY')
-
-DOMAIN_NAME = socket.gethostname()
-TRUENAS_ADDRESS = 'localhost'
-VERIFY = False
-PRIVATEKEY_PATH = os.getenv('CERTS_DEPLOY_PRIVATE_KEY_PATH')
-FULLCHAIN_PATH = os.getenv('CERTS_DEPLOY_FULLCHAIN_PATH')
-PROTOCOL = 'http://'
-PORT = '80'
-FTP_ENABLED = bool(os.getenv('CERTS_DEPLOY_FTP_ENABLED', ''))
-S3_ENABLED = bool(os.getenv('CERTS_DEPLOY_S3_ENABLED', ''))
-now = datetime.now()
-cert = "letsencrypt-%s-%s-%s-%s" %(now.year, now.strftime('%m'), now.strftime('%d'), ''.join(c for c in now.strftime('%X') if
-c.isdigit()))
-
-
-# Set some general request params
-session = requests.Session()
-session.headers.update({
-  'Content-Type': 'application/json'
-})
-if API_KEY:
-  session.headers.update({
-    'Authorization': f'Bearer {API_KEY}'
-  })
-else:
-  print ("Unable to authenticate. Specify 'CERTS_DEPLOY_API_KEY' in the os Env.")
-  exit(1)
-if not PRIVATEKEY_PATH:
-  print ("Unable to find private key. Specify 'CERTS_DEPLOY_PRIVATE_KEY_PATH' in the os Env.")
-  exit(1)
-if not FULLCHAIN_PATH:
-  print ("Unable to find private key. Specify 'CERTS_DEPLOY_FULLCHAIN_PATH' in the os Env.")
-  exit(1)
-
-# Load cert/key
-with open(PRIVATEKEY_PATH, 'r') as file:
-  priv_key = file.read()
-with open(FULLCHAIN_PATH, 'r') as file:
-  full_chain = file.read()
-
-# Update or create certificate
-r = session.post(
-  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/',
-  verify=VERIFY,
-  data=json.dumps({
-    "create_type": "CERTIFICATE_CREATE_IMPORTED",
-    "name": cert,
-    "certificate": full_chain,
-    "privatekey": priv_key,
-  })
-)
-
-if r.status_code == 200:
-  print ("Certificate import successful")
-else:
-  print ("Error importing certificate!")
-  print (r.text)
-  sys.exit(1)
-
-# Sleep for a few seconds to let the cert propagate
-time.sleep(5)
-
-# Download certificate list
-limit = {'limit': 0} # set limit to 0 to disable paging in the event of many certificates
-r = session.get(
-  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/',
-  verify=VERIFY,
-  params=limit
-)
-
-if r.status_code == 200:
-  print ("Certificate list successful")
-else:
-  print ("Error listing certificates!")
-  print (r.text)
-  sys.exit(1)
-
-# Parse certificate list to find the id that matches our cert name
-cert_list = r.json()
-
-new_cert_data = None
-for cert_data in cert_list:
-  if cert_data['name'] == cert:
-    new_cert_data = cert_data
-    cert_id = new_cert_data['id']
-    break
-
-if not new_cert_data:
-  print ("Error searching for newly imported certificate in certificate list.")
-  sys.exit(1)
-
-# Set our cert as active
-r = session.put(
-  PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/system/general/',
-  verify=VERIFY,
-  data=json.dumps({
-    "ui_certificate": cert_id,
-  })
-)
-
-if r.status_code == 200:
-  print ("Setting active certificate successful")
-else:
-  print ("Error setting active certificate!")
-  print (r.text)
-  sys.exit(1)
-
-if FTP_ENABLED:
-  # Set our cert as active for FTP plugin
-  r = session.put(
-    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/ftp/',
-    verify=VERIFY,
-    data=json.dumps({
-      "ssltls_certfile": cert,
-    }),
-  )
-
-  if r.status_code == 200:
-    print ("Setting active FTP certificate successful")
-  else:
-    print ("Error setting active FTP certificate!")
-    print (r.text)
-    sys.exit(1)
-
-if S3_ENABLED:
-  # Set our cert as active for S3 plugin
-  r = session.put(
-    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/s3/',
-    verify=VERIFY,
-    data=json.dumps({
-      "certificate": cert_id,
-    }),
-  )
-
-  if r.status_code == 200:
-    print ("Setting active S3 certificate successful")
-  else:
-    print ("Error setting active S3 certificate!")
-    print (r)
-    sys.exit(1)
-
-# Get expired and old certs with same SAN
-cert_ids_same_san = set()
-cert_ids_expired = set()
-for cert_data in cert_list:
-  if set(cert_data['san']) == set(new_cert_data['san']):
-      cert_ids_same_san.add(cert_data['id'])
-
-  issued_date = datetime.strptime(cert_data['from'], "%c")
-  lifetime = timedelta(days=cert_data['lifetime'])
-  expiration_date = issued_date + lifetime
-  if expiration_date < now:
-      cert_ids_expired.add(cert_data['id'])
-
-# Remove new cert_id from lists
-if cert_id in cert_ids_expired:
-  cert_ids_expired.remove(cert_id)
-
-if cert_id in cert_ids_same_san:
-  cert_ids_same_san.remove(cert_id)
-
-# Delete expired and old certificates with same SAN from freenas
-for cid in (cert_ids_same_san | cert_ids_expired):
-  r = session.delete(
-    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/certificate/id/' + str(cid),
-    verify=VERIFY
-  )
-
-  for c in cert_list:
-    if c['id'] == cid:
-      cert_name = c['name']
-
-  if r.status_code == 200:
-    print ("Deleting certificate " + cert_name + " successful")
-  else:
-    print ("Error deleting certificate " + cert_name + "!")
-    print (r.text)
-    sys.exit(1)
-
-# Reload nginx with new cert
-# If everything goes right, the request fails with a ConnectionError
-try:
-  r = session.post(
-    PROTOCOL + TRUENAS_ADDRESS + ':' + PORT + '/api/v2.0/system/general/ui_restart',
-    verify=VERIFY
-  )
-
-  if r.status_code == 200:
-    print ("Reloading WebUI successful")
-    print ("deploy_freenas.py executed successfully")
-  else:
-    print ("Error reloading WebUI!")
-    print ("{}: {}".format(r.status_code, r.text))
-    sys.exit(1)
-except requests.exceptions.ConnectionError:
-    print ("Error reloading WebUI!")
-    sys.exit(1)

kubernetes/apps/default/homelab/truenas/certs-deploy/truenas-certs-deploy.sh

@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-
-set -o nounset
-set -o errexit
-
-mkdir -p ~/.ssh
-cp /opt/id_rsa ~/.ssh/id_rsa
-chmod 600 ~/.ssh/id_rsa
-
-if [ "${HOSTNAME}" == "truenas" ]; then
-    printf -v truenas_api_key %q "$TRUENAS_API_KEY"
-elif [ "${HOSTNAME}" == "truenas-remote" ]; then
-    printf -v truenas_api_key %q "$TRUENAS_REMOTE_API_KEY"
-fi
-printf -v cert_deploy_minio_enabled_str %q "$CERTS_DEPLOY_MINIO_ENABLED"
-printf -v cert_deploy_postgresql_enabled_str %q "$CERTS_DEPLOY_POSTGRESQL_ENABLED"
-printf -v pushover_api_token_str %q "$PUSHOVER_API_TOKEN"
-printf -v pushover_user_key_str %q "$PUSHOVER_USER_KEY"
-printf -v secret_domain_str %q "$SECRET_DOMAIN"
-
-scp -o StrictHostKeyChecking=no /app/truenas-certs-deploy.py homelab@${HOSTNAME}.${SECRET_DOMAIN}:${TRUENAS_HOME}/scripts/certificates_deploy.py
-
-ssh -o StrictHostKeyChecking=no homelab@${HOSTNAME}.${SECRET_DOMAIN} "/bin/bash -s $truenas_api_key $cert_deploy_minio_enabled_str $cert_deploy_postgresql_enabled_str $pushover_api_token_str $pushover_user_key_str $secret_domain_str" << 'EOF'
-
-set -o nounset
-set -o errexit
-
-PUSHOVER_API_TOKEN=$4
-PUSHOVER_USER_KEY=$5
-SECRET_DOMAIN=$6
-
-# Variables
-TARGET=$(hostname)
-DAYS=21
-CERTIFICATE_PATH="${HOME}/letsencrypt/${SECRET_DOMAIN}"
-SCRIPT_PATH="${HOME}/scripts"
-
-export CERTS_DEPLOY_API_KEY=$1
-export CERTS_DEPLOY_PRIVATE_KEY_PATH=${CERTIFICATE_PATH}/key.pem
-export CERTS_DEPLOY_FULLCHAIN_PATH=${CERTIFICATE_PATH}/fullchain.pem
-if [ "$2" == "True" ]; then
-    export CERTS_DEPLOY_MINIO_ENABLED=$2
-fi
-CERTS_DEPLOY_MINIO_CERT_PATH=/mnt/storage/iocage/jails/minio_v2/root/home/minio/certs
-if [ "$3" == "True" ]; then
-    export CERTS_DEPLOY_POSTGRESQL_ENABLED=$3
-fi
-CERTS_DEPLOY_POSTGRESQL_PATH=/mnt/apps/postgresql
-
-# Check if cert is older than 69 days
-result=$(find ${CERTS_DEPLOY_PRIVATE_KEY_PATH} -mtime +69)
-
-if [[ "$result" == "${CERTS_DEPLOY_PRIVATE_KEY_PATH}" ]]; then
-    echo "ERROR - Certificate is older than 69 days"
-    echo "ERROR - Verify than it has been renewed by ACME client on opnsense and that the upload automation has been executed"
-    curl -s \
-        --form-string "token=${PUSHOVER_API_TOKEN}" \
-        --form-string "user=${PUSHOVER_USER_KEY}" \
-        --form-string "message=Certificate on $TARGET is older than 69 days. Verify than it has been renewed by ACME client on opnsense and that the upload automation has been executed" \
-        https://api.pushover.net/1/messages.json
-else
-    echo "INFO checking if $TARGET expires in less than $DAYS days"
-    set +o errexit
-    openssl x509 -checkend $(( 24*3600*$DAYS )) -noout -in <(openssl s_client -showcerts -connect $TARGET:443 </dev/null 2>/dev/null | openssl x509 -outform PEM)
-    if [[ $? -ne 0 ]]; then
-        set -o errexit
-        echo "INFO - Certificate expires in less than $DAYS days"
-        echo "INFO - Deploying new certificate"
-        # Deploy certificate (truenas UI)
-        python ${SCRIPT_PATH}/certificates_deploy.py
-        # Copy certificates (minio)
-        if [ "CERTS_DEPLOY_MINIO_ENABLED" == "True" ]; then
-            cp -pr ${CERTS_DEPLOY_PRIVATE_KEY_PATH} ${CERTS_DEPLOY_MINIO_CERT_PATH}/private.key
-            cp -pr ${CERTS_DEPLOY_FULLCHAIN_PATH} ${CERTS_DEPLOY_MINIO_CERT_PATH}/public.crt
-            iocage exec minio_v2 'service minio restart'
-        fi
-        # Copy certificates (postgresql)
-        if [ "CERTS_DEPLOY_POSTGRESQL_ENABLED" == "True" ]; then
-            pg_data_dirs=$(find /mnt/{{ postgresql_pool_name }}/postgresql -type d -maxdepth 1 -name '*data*' -exec basename {} \;)
-            for i in $pg_data_dirs; do
-                cp -pr ${CERTS_DEPLOY_PRIVATE_KEY_PATH} ${CERTS_DEPLOY_POSTGRESQL_PATH}/$i/server.key
-                cp -pr ${CERTS_DEPLOY_FULLCHAIN_PATH} ${CERTS_DEPLOY_POSTGRESQL_PATH}/$i/server.crt
-                iocage exec postgresql_v${i: -2} 'service postgresql reload'
-            done
-        fi
-        curl -s \
-            --form-string "token=${PUSHOVER_API_TOKEN}" \
-            --form-string "user=${PUSHOVER_USER_KEY}" \
-            --form-string "message=New Let's Encrypt certificate deployed on $TARGET." \
-            https://api.pushover.net/1/messages.json
-
-    else
-        echo "INFO - Certificate expires in more than $DAYS days"
-    fi
-fi
-
-EOF

kubernetes/apps/default/homelab/truenas/kustomization.yaml

@@ -5,7 +5,6 @@ kind: Kustomization
 namespace: default
 resources:
   - ./backup
-  - ./certs-deploy
   - ./externalsecret.yaml
   - ./photo-sort
   - ../../../../templates/gatus/guarded

kubernetes/apps/default/homelab/truenas/photo-sort/helmrelease.yaml

@@ -50,8 +50,8 @@ spec:
 
                 /app/sort.sh
             env:
-              SORT_SOURCE_DIR: /mnt/storage/photo/mobile
-              SORT_DEST_DIR: /mnt/storage/photo
+              SORT_SOURCE_DIR: /var/mnt/vol1/photo/mobile
+              SORT_DEST_DIR: /var/mnt/vol1/photo
     service:
       app:
         controller: *app
@@ -67,7 +67,7 @@ spec:
             readOnly: true
       photo:
         type: nfs
-        path: /mnt/storage/photo
+        path: /var/mnt/vol1/photo
         server: 192.168.9.10
         globalMounts:
-          - path: /mnt/storage/photo
+          - path: /var/mnt/vol1/photo