feat: migrate thanos to another chart

kubernetes/apps/monitoring/thanos/app/externalsecret.yaml

@@ -1,21 +0,0 @@
----
-# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/external-secrets.io/externalsecret_v1beta1.json
-apiVersion: external-secrets.io/v1beta1
-kind: ExternalSecret
-metadata:
-  name: thanos
-  namespace: flux-system
-spec:
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: onepassword-connect
-  target:
-    name: thanos-secret
-    template:
-      engineVersion: v2
-      data:
-        S3_ACCESS_KEY: "{{ .THANOS_S3_ACCESS_KEY }}"
-        S3_SECRET_KEY: "{{ .THANOS_S3_SECRET_KEY }}"
-  dataFrom:
-    - extract:
-        key: thanos

kubernetes/apps/monitoring/thanos/app/helmrelease.yaml

@@ -7,122 +7,113 @@ metadata:
   namespace: monitoring
 spec:
   interval: 30m
+  timeout: 15m
   chart:
     spec:
       chart: thanos
-      version: 15.4.4
+      version: 1.17.0
       sourceRef:
         kind: HelmRepository
-        name: bitnami
+        name: stevehipwell
         namespace: flux-system
-  maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
     cleanupOnFail: true
     remediation:
+      strategy: rollback
       retries: 3
-  uninstall:
-    keepHistory: false
+  dependsOn:
+    - name: openebs
+      namespace: openebs-system
+    - name: rook-ceph-cluster
+      namespace: rook-ceph
+  valuesFrom:
+    - targetPath: objstoreConfig.value.config.bucket
+      kind: ConfigMap
+      name: thanos-bucket
+      valuesKey: BUCKET_NAME
+    - targetPath: objstoreConfig.value.config.endpoint
+      kind: ConfigMap
+      name: thanos-bucket
+      valuesKey: BUCKET_HOST
+    - targetPath: objstoreConfig.value.config.region
+      kind: ConfigMap
+      name: thanos-bucket
+      valuesKey: BUCKET_REGION
+    - targetPath: objstoreConfig.value.config.access_key
+      kind: Secret
+      name: thanos-bucket
+      valuesKey: AWS_ACCESS_KEY_ID
+    - targetPath: objstoreConfig.value.config.secret_key
+      kind: Secret
+      name: thanos-bucket
+      valuesKey: AWS_SECRET_ACCESS_KEY
   values:
-    image:
-      registry: quay.io
-      repository: thanos/thanos
-      tag: v0.35.0@sha256:fa1d28718df00b68d6ad85d7c7d4703bd9f59e5cd8be8da6540ea398cf701a1f
     objstoreConfig:
-      type: s3
-      config:
-        bucket: thanos
-        endpoint: "s3.${SECRET_INTERNAL_DOMAIN}"
-        region: ""
-        # insecure: true
-    query:
+      value:
+        type: s3
+        config:
+          insecure: true
+    additionalEndpoints:
+      - dnssrv+_grpc._tcp.kube-prometheus-stack-thanos-discovery.monitoring.svc.cluster.local
+    additionalReplicaLabels: ["__replica__"]
+    serviceMonitor:
       enabled: true
-      replicaCount: 2
-      podAntiAffinityPreset: hard
-      replicaLabels:
-        - replica
-      dnsDiscovery:
-        sidecarsService: kube-prometheus-stack-thanos-discovery
-        sidecarsNamespace: monitoring
-        stores:
-          - "dnssrv+_grpc._tcp.kube-prometheus-stack-thanos-discovery"
-          - "thanos-store.${SECRET_DOMAIN}:443"
-      ingress:
+    compact:
+      enabled: true
+      extraArgs:
+        - --compact.concurrency=4
+        - --delete-delay=30m
+        - --retention.resolution-raw=14d
+        - --retention.resolution-5m=30d
+        - --retention.resolution-1h=60d
+      persistence: &persistence
         enabled: true
-        hostname: &host "thanos-query.${SECRET_CLUSTER_DOMAIN}"
-        annotations:
-          nginx.ingress.kubernetes.io/auth-method: GET
-          nginx.ingress.kubernetes.io/auth-url: http://authelia.default.svc.cluster.local.:8888/api/verify
-          nginx.ingress.kubernetes.io/auth-signin: https://auth.${SECRET_CLUSTER_DOMAIN}?rm=$request_method
-          nginx.ingress.kubernetes.io/auth-response-headers: Remote-User,Remote-Name,Remote-Groups,Remote-Email
-          nginx.ingress.kubernetes.io/auth-snippet: proxy_set_header X-Forwarded-Method $request_method;
-          hajimari.io/enable: "false"
-        ingressClassName: "nginx"
-        tls: true
-        extraTls:
-          - hosts:
-              - *host
-      resources:
-        requests:
-          cpu: 15m
-          memory: 64M
-        limits:
-          memory: 99M
+        storageClass: openebs-hostpath
+        size: 10Gi
+    query:
+      replicas: 3
+      extraArgs: ["--alert.query-url=https://thanos.${SECRET_CLUSTER_DOMAIN}"]
+      additionalStores: ["thanos.turbo.ac:10901"]
     queryFrontend:
       enabled: true
-    bucketweb:
-      enabled: true
-      refresh: "10m"
-    compactor:
-      enabled: true
-      extraFlags:
-        - "--compact.concurrency"
-        - "4"
-      retentionResolutionRaw: 14d
-      retentionResolution5m: 14d
-      retentionResolution1h: 30d
+      replicas: 3
+      extraEnv: &extraEnv
+        - name: THANOS_CACHE_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: &configMap thanos-cache-configmap
+              key: cache.yaml
+      extraArgs: ["--query-range.response-cache-config=$(THANOS_CACHE_CONFIG)"]
       ingress:
         enabled: true
-        hostname: &host "thanos-compactor.${SECRET_CLUSTER_DOMAIN}"
-        ingressClassName: "nginx"
-        annotations:
-          hajimari.io/enable: "false"
-        tls: true
-        extraTls:
-          - hosts:
-              - *host
-      persistence:
-        enabled: true
-        storageClass: "rook-ceph-block"
-        size: 15Gi
-      resourcesPreset: small
-    storegateway:
+        ingressClassName: internal
+        hosts:
+          - thanos.devbu.io
+      podAnnotations: &podAnnotations
+        configmap.reloader.stakater.com/reload: *configMap
+    rule:
       enabled: true
-      resources:
-        requests:
-          cpu: 23m
-          memory: 204M
-        limits:
-          memory: 226M
-      persistence:
-        enabled: true
-        storageClass: "rook-ceph-block"
-        size: 4Gi
-    ruler:
-      enabled: false
-    metrics:
-      enabled: true
-      serviceMonitor:
-        enabled: true
-  valuesFrom:
-    - kind: Secret
-      name: thanos-secret
-      valuesKey: S3_ACCESS_KEY
-      targetPath: objstoreConfig.config.access_key
-    - kind: Secret
-      name: thanos-secret
-      valuesKey: S3_SECRET_KEY
-      targetPath: objstoreConfig.config.secret_key
+      replicas: 3
+      extraArgs: ["--web.prefix-header=X-Forwarded-Prefix"]
+      alertmanagersConfig:
+        value: |-
+          alertmanagers:
+            - api_version: v2
+              static_configs:
+                - dnssrv+_http-web._tcp.alertmanager-operated.monitoring.svc.cluster.local
+      rules:
+        value: |-
+          groups:
+            - name: PrometheusWatcher
+              rules:
+                - alert: PrometheusDown
+                  annotations:
+                    summary: A Prometheus has disappeared from Prometheus target discovery
+                  expr: absent(up{job="kube-prometheus-stack-prometheus"})
+                  for: 5m
+                  labels:
+                    severity: critical
+      persistence: *persistence

kubernetes/apps/monitoring/thanos/app/kustomization.yaml

@@ -4,5 +4,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: monitoring
 resources:
-  - ./externalsecret.yaml
+  - ./objectbucketclaim.yaml
   - ./helmrelease.yaml
+  - ./pushsecret.yaml
+configMapGenerator:
+  - name: thanos-cache-configmap
+    files:
+      - cache.yaml=./resources/cache.yaml
+generatorOptions:
+  disableNameSuffixHash: true

kubernetes/apps/monitoring/thanos/app/objectbucketclaim.yaml

@@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/objectbucket.io/objectbucketclaim_v1alpha1.json
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: thanos-bucket
+spec:
+  bucketName: thanos
+  storageClassName: rook-ceph-bucket

kubernetes/apps/monitoring/thanos/app/pushsecret.yaml

@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/external-secrets.io/pushsecret_v1alpha1.json
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: thanos
+spec:
+  refreshInterval: 1h
+  secretStoreRefs:
+    - name: onepassword-connect
+      kind: ClusterSecretStore
+  selector:
+    secret:
+      name: thanos-bucket
+  data:
+    - match:
+        secretKey: &key AWS_ACCESS_KEY_ID
+        remoteRef:
+          remoteKey: thanos
+          property: *key
+    - match:
+        secretKey: &key AWS_SECRET_ACCESS_KEY
+        remoteRef:
+          remoteKey: thanos
+          property: *key

kubernetes/apps/monitoring/thanos/app/resources/cache.yaml

@@ -0,0 +1,5 @@
+---
+type: REDIS
+config:
+  addr: dragonfly.database.svc.cluster.local:6379
+  db: 1