diff --git a/ansible/inventory/group_vars/worker/k3s.yml b/ansible/inventory/group_vars/worker/k3s.yml index 8c7e94c80..152e4a103 100644 --- a/ansible/inventory/group_vars/worker/k3s.yml +++ b/ansible/inventory/group_vars/worker/k3s.yml @@ -14,3 +14,5 @@ k3s_agent: # Allow pods to be rescheduled quicker in the case of a node failure # https://github.com/k3s-io/k3s/issues/1264 - "node-status-update-frequency=4s" + node-label: + - node-role.kubernetes.io/worker=true diff --git a/cluster/apps/authentication/authelia/helm-release.yaml b/cluster/apps/authentication/authelia/helm-release.yaml index 37b312779..592ef43f1 100644 --- a/cluster/apps/authentication/authelia/helm-release.yaml +++ b/cluster/apps/authentication/authelia/helm-release.yaml @@ -50,8 +50,7 @@ spec: enabled: true ingressClassName: "nginx" annotations: - external-dns.alpha.kubernetes.io/target: "services.${SECRET_DOMAIN}." - external-dns/is-public: "true" + external-dns.home.arpa/enabled: "true" nginx.ingress.kubernetes.io/configuration-snippet: | add_header Cache-Control "no-store"; add_header Pragma "no-cache"; diff --git a/cluster/apps/downloaders/sabnzbd/helm-release.yaml b/cluster/apps/downloaders/sabnzbd/helm-release.yaml index 9b180b3c3..8cb464cb0 100644 --- a/cluster/apps/downloaders/sabnzbd/helm-release.yaml +++ b/cluster/apps/downloaders/sabnzbd/helm-release.yaml @@ -48,8 +48,7 @@ spec: enabled: true ingressClassName: "nginx" annotations: - nginx.ingress.kubernetes.io/auth-url: "http://authelia.default.svc.cluster.local/api/verify" - nginx.ingress.kubernetes.io/auth-signin: "https://auth.${SECRET_CLUSTER_DOMAIN}" + auth.home.arpa/enabled: "true" nginx.ingress.kubernetes.io/configuration-snippet: | proxy_set_header Accept-Encoding ""; sub_filter '' ''; diff --git a/cluster/core/kube-system/coredns-nodecache/configmap.yaml b/cluster/apps/kube-tools/coredns-nodecache/configmap.yaml similarity index 97% rename from cluster/core/kube-system/coredns-nodecache/configmap.yaml rename to cluster/apps/kube-tools/coredns-nodecache/configmap.yaml index 9686f75ad..cf0a29f78 100644 --- a/cluster/core/kube-system/coredns-nodecache/configmap.yaml +++ b/cluster/apps/kube-tools/coredns-nodecache/configmap.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: coredns-nodecache-primary - namespace: kube-system + namespace: default data: Corefile: | cluster.local:53 { @@ -65,7 +65,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: coredns-nodecache-secondary - namespace: kube-system + namespace: default data: Corefile: | cluster.local:53 { diff --git a/cluster/core/kube-system/coredns-nodecache/daemonset.yaml b/cluster/apps/kube-tools/coredns-nodecache/daemonset.yaml similarity index 100% rename from cluster/core/kube-system/coredns-nodecache/daemonset.yaml rename to cluster/apps/kube-tools/coredns-nodecache/daemonset.yaml diff --git a/cluster/core/kube-system/coredns-nodecache/kustomization.yaml b/cluster/apps/kube-tools/coredns-nodecache/kustomization.yaml similarity index 100% rename from cluster/core/kube-system/coredns-nodecache/kustomization.yaml rename to cluster/apps/kube-tools/coredns-nodecache/kustomization.yaml diff --git a/cluster/core/kube-system/coredns-nodecache/service-account.yaml b/cluster/apps/kube-tools/coredns-nodecache/service-account.yaml similarity index 100% rename from cluster/core/kube-system/coredns-nodecache/service-account.yaml rename to cluster/apps/kube-tools/coredns-nodecache/service-account.yaml diff --git a/cluster/apps/kube-tools/descheduler/helm-release.yaml b/cluster/apps/kube-tools/descheduler/helm-release.yaml new file mode 100644 index 000000000..40f5e4f11 --- /dev/null +++ b/cluster/apps/kube-tools/descheduler/helm-release.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app descheduler + namespace: default +spec: + interval: 15m + chart: + spec: + chart: *app + version: 0.24.1 + sourceRef: + kind: HelmRepository + name: descheduler-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + kind: Deployment + replicas: 1 + leaderElection: + enabled: true + leaseDuration: 15s + renewDeadline: 10s + retryPeriod: 2s + resourceLock: "leases" + resourceName: "descheduler" + resourceNamescape: "kube-system" + deschedulerPolicy: + strategies: + RemoveDuplicates: + enabled: true + RemovePodsViolatingNodeTaints: + enabled: true + RemovePodsViolatingNodeAffinity: + enabled: true + params: + nodeAffinityType: + - requiredDuringSchedulingIgnoredDuringExecution + RemovePodsViolatingTopologySpreadConstraint: + enabled: true + params: + includeSoftConstraints: true + RemovePodsViolatingInterPodAntiAffinity: + enabled: true + params: + nodeFit: true + LowNodeUtilization: + enabled: false + RemoveFailedPods: + enabled: true + params: + failedPods: + includingInitContainers: true + excludeOwnerKinds: + - "Job" + minPodLifetimeSeconds: 3600 + RemovePodsHavingTooManyRestarts: + enabled: true + params: + podsHavingTooManyRestarts: + podRestartThreshold: 100 + includingInitContainers: true + service: + enabled: true + serviceMonitor: + enabled: true + podAnnotations: + configmap.reloader.stakater.com/reload: *app + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [*app] + topologyKey: kubernetes.io/hostname diff --git a/cluster/core/kube-system/node-feature-discovery/kustomization.yaml b/cluster/apps/kube-tools/descheduler/kustomization.yaml similarity index 96% rename from cluster/core/kube-system/node-feature-discovery/kustomization.yaml rename to cluster/apps/kube-tools/descheduler/kustomization.yaml index 34a8531ce..2fa2de20c 100644 --- a/cluster/core/kube-system/node-feature-discovery/kustomization.yaml +++ b/cluster/apps/kube-tools/descheduler/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/kube-tools/goldilocks/helm-release.yaml b/cluster/apps/kube-tools/goldilocks/helm-release.yaml new file mode 100644 index 000000000..262ba40d9 --- /dev/null +++ b/cluster/apps/kube-tools/goldilocks/helm-release.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: goldilocks + namespace: default +spec: + interval: 15m + chart: + spec: + chart: goldilocks + version: 6.2.0 + sourceRef: + kind: HelmRepository + name: fairwinds-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + controller: + resources: + requests: + cpu: 10m + memory: 64M + limits: + memory: 250M + dashboard: + replicaCount: 1 + ingress: + enabled: true + ingressClassName: "nginx" + hosts: + - host: &host "goldilocks.${SECRET_CLUSTER_DOMAIN}" + paths: + - path: / + type: Prefix + tls: + - hosts: + - *host + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + memory: 150Mi diff --git a/cluster/core/kube-system/intel-gpu-plugin/kustomization.yaml b/cluster/apps/kube-tools/goldilocks/kustomization.yaml similarity index 96% rename from cluster/core/kube-system/intel-gpu-plugin/kustomization.yaml rename to cluster/apps/kube-tools/goldilocks/kustomization.yaml index 34a8531ce..2fa2de20c 100644 --- a/cluster/core/kube-system/intel-gpu-plugin/kustomization.yaml +++ b/cluster/apps/kube-tools/goldilocks/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/kube-tools/intel-gpu-exporter/helm-release.yaml b/cluster/apps/kube-tools/intel-gpu-exporter/helm-release.yaml new file mode 100644 index 000000000..d70e41987 --- /dev/null +++ b/cluster/apps/kube-tools/intel-gpu-exporter/helm-release.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app intel-gpu-exporter + namespace: default +spec: + interval: 15m + chart: + spec: + chart: app-template + version: 0.1.1 + sourceRef: + kind: HelmRepository + name: bjw-s-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + dependsOn: + - name: intel-gpu-plugin + namespace: default + values: + controller: + type: daemonset + image: + repository: ghcr.io/onedr0p/intel-gpu-exporter + tag: rolling@sha256:1c84020b442e0f95e2e6a46281d3bfc6199902d3d91b10515fc000e43c9a7421 + service: + main: + ports: + http: + port: 8080 + securityContext: + privileged: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: feature.node.kubernetes.io/custom-intel-gpu + operator: In + values: + - "true" + resources: + requests: + gpu.intel.com/i915: 1 + cpu: 100m + memory: 100Mi + limits: + gpu.intel.com/i915: 1 + memory: 500Mi diff --git a/cluster/apps/kube-tools/intel-gpu-exporter/kustomization.yaml b/cluster/apps/kube-tools/intel-gpu-exporter/kustomization.yaml new file mode 100644 index 000000000..d4eef3d12 --- /dev/null +++ b/cluster/apps/kube-tools/intel-gpu-exporter/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-release.yaml + - service-monitor.yaml diff --git a/cluster/apps/kube-tools/intel-gpu-exporter/service-monitor.yaml b/cluster/apps/kube-tools/intel-gpu-exporter/service-monitor.yaml new file mode 100644 index 000000000..3c75fdac4 --- /dev/null +++ b/cluster/apps/kube-tools/intel-gpu-exporter/service-monitor.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: &app intel-gpu-exporter + namespace: default + labels: &labels + app.kubernetes.io/instance: *app + app.kubernetes.io/name: *app +spec: + selector: + matchLabels: + <<: *labels + endpoints: + - port: http + interval: 1m + scrapeTimeout: 10s + path: /metrics + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node diff --git a/cluster/apps/kube-tools/intel-gpu-plugin/helm-release.yaml b/cluster/apps/kube-tools/intel-gpu-plugin/helm-release.yaml new file mode 100644 index 000000000..7a4be0d9c --- /dev/null +++ b/cluster/apps/kube-tools/intel-gpu-plugin/helm-release.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app intel-gpu-plugin + namespace: default +spec: + interval: 15m + chart: + spec: + chart: app-template + version: 0.1.1 + sourceRef: + kind: HelmRepository + name: bjw-s-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + dependsOn: + - name: node-feature-discovery + namespace: default + values: + controller: + type: daemonset + strategy: RollingUpdate + image: + repository: docker.io/intel/intel-gpu-plugin + tag: 0.24.0 + pullPolicy: IfNotPresent + args: + - -shared-dev-num + - "4" + service: + main: + enabled: false + # TODO(intel-gpu-plugin): Write probes to check for something to tell if it's working + probes: + liveness: + enabled: false + readiness: + enabled: false + startup: + enabled: false + persistence: + devfs: + enabled: true + type: hostPath + hostPath: /dev/dri + hostPathType: Directory + readOnly: true + sysfs: + enabled: true + type: hostPath + hostPath: /sys/class/drm + hostPathType: Directory + readOnly: true + kubeletsockets: + enabled: true + type: hostPath + hostPathType: Directory + hostPath: /var/lib/kubelet/device-plugins + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: feature.node.kubernetes.io/custom-intel-gpu + operator: In + values: + - "true" diff --git a/cluster/core/kube-system/reloader/kustomization.yaml b/cluster/apps/kube-tools/intel-gpu-plugin/kustomization.yaml similarity index 96% rename from cluster/core/kube-system/reloader/kustomization.yaml rename to cluster/apps/kube-tools/intel-gpu-plugin/kustomization.yaml index 34a8531ce..2fa2de20c 100644 --- a/cluster/core/kube-system/reloader/kustomization.yaml +++ b/cluster/apps/kube-tools/intel-gpu-plugin/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/kube-system/kured/helm-release.yaml b/cluster/apps/kube-tools/kured/helm-release.yaml similarity index 91% rename from cluster/core/kube-system/kured/helm-release.yaml rename to cluster/apps/kube-tools/kured/helm-release.yaml index 0030e4919..2f7d1c5d2 100644 --- a/cluster/core/kube-system/kured/helm-release.yaml +++ b/cluster/apps/kube-tools/kured/helm-release.yaml @@ -8,7 +8,6 @@ spec: interval: 5m chart: spec: - # renovate: registryUrl=https://weaveworks.github.io/kured chart: kured version: 3.0.1 sourceRef: diff --git a/cluster/core/kube-system/kured/kustomization.yaml b/cluster/apps/kube-tools/kured/kustomization.yaml similarity index 100% rename from cluster/core/kube-system/kured/kustomization.yaml rename to cluster/apps/kube-tools/kured/kustomization.yaml diff --git a/cluster/core/kube-system/kured/prometheus-rule.yaml b/cluster/apps/kube-tools/kured/prometheus-rule.yaml similarity index 100% rename from cluster/core/kube-system/kured/prometheus-rule.yaml rename to cluster/apps/kube-tools/kured/prometheus-rule.yaml diff --git a/cluster/core/kube-system/kustomization.yaml b/cluster/apps/kube-tools/kustomization.yaml similarity index 56% rename from cluster/core/kube-system/kustomization.yaml rename to cluster/apps/kube-tools/kustomization.yaml index 4ae9e6df3..db97f6f67 100644 --- a/cluster/core/kube-system/kustomization.yaml +++ b/cluster/apps/kube-tools/kustomization.yaml @@ -1,10 +1,16 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - namespace.yaml - - coredns-nodecache + #- coredens-nodecache - descheduler + - goldilocks + - intel-gpu-exporter - intel-gpu-plugin - kured + - kyverno + - metrics-server - node-feature-discovery - reloader + - system-upgrade + - vpa diff --git a/cluster/apps/kube-tools/kyverno/helm-release.yaml b/cluster/apps/kube-tools/kyverno/helm-release.yaml new file mode 100644 index 000000000..25077fe2b --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/helm-release.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kyverno + namespace: kyverno +spec: + interval: 15m + chart: + spec: + chart: kyverno + version: v2.5.3 + sourceRef: + kind: HelmRepository + name: kyverno-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + installCRDs: false + replicaCount: 3 + extraArgs: + - --autogenInternals=false + - --clientRateLimitQPS=30 + - --clientRateLimitBurst=60 + serviceMonitor: + enabled: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: kyverno diff --git a/cluster/apps/kube-tools/kyverno/kustomization.yaml b/cluster/apps/kube-tools/kyverno/kustomization.yaml new file mode 100644 index 000000000..7a789e523 --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # renovate: registryUrl=https://kyverno.github.io/kyverno/ chart=kyverno + - github.com/kyverno/kyverno//config/crds?ref=kyverno-chart-v2.5.3 + - helm-release.yaml + - rbac.yaml + - policies diff --git a/cluster/apps/kube-tools/kyverno/policies/apply-ingress-auth-annotations.yaml b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-auth-annotations.yaml new file mode 100644 index 000000000..5554ea83d --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-auth-annotations.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: apply-ingress-auth-annotations + annotations: + policies.kyverno.io/title: Apply Ingress Auth Annotations + policies.kyverno.io/subject: Ingress + policies.kyverno.io/description: >- + This policy creates auth annotations on ingresses. When + the `auth.home.arpa/enabled` annotation is `true` it + applies the nginx auth annotations for use with Authelia. +spec: + mutateExistingOnPolicyUpdate: true + generateExistingOnPolicyUpdate: true + rules: + - name: auth + match: + any: + - resources: + kinds: ["Ingress"] + annotations: + auth.home.arpa/enabled: "true" + mutate: + patchStrategicMerge: + metadata: + annotations: + +(nginx.ingress.kubernetes.io/auth-method): GET + +(nginx.ingress.kubernetes.io/auth-url): |- + http://authelia.default.svc.cluster.local/api/verify + +(nginx.ingress.kubernetes.io/auth-signin): |- + https://auth.${SECRET_CLUSTER_DOMAIN}?rm=$request_method + +(nginx.ingress.kubernetes.io/auth-response-headers): |- + Remote-User,Remote-Name,Remote-Groups,Remote-Email + +(nginx.ingress.kubernetes.io/auth-snippet): | + proxy_set_header X-Forwarded-Method $request_method; diff --git a/cluster/apps/kube-tools/kyverno/policies/apply-ingress-external-dns-annotations.yaml b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-external-dns-annotations.yaml new file mode 100644 index 000000000..fcc632772 --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-external-dns-annotations.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: apply-ingress-external-dns-annotations + annotations: + policies.kyverno.io/title: Apply Ingress External-DNS Annotations + policies.kyverno.io/subject: Ingress + policies.kyverno.io/description: >- + This policy creates external-dns annotations on ingresses. + When the `external-dns.home.arpa/enabled` annotation is `true` + it applies the external-dns annotations for use with external + application access. +spec: + mutateExistingOnPolicyUpdate: true + generateExistingOnPolicyUpdate: true + rules: + - name: external-dns + match: + any: + - resources: + kinds: ["Ingress"] + annotations: + external-dns.home.arpa/enabled: "true" + mutate: + patchStrategicMerge: + metadata: + annotations: + +(external-dns.alpha.kubernetes.io/target): |- + services.${SECRET_DOMAIN}. + +(external-dns/is-public): |- + true diff --git a/cluster/apps/kube-tools/kyverno/policies/apply-ingress-whitelist-annotations.yaml b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-whitelist-annotations.yaml new file mode 100644 index 000000000..7a5e39a4a --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/apply-ingress-whitelist-annotations.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: apply-ingress-whitelist-annotations + annotations: + policies.kyverno.io/title: Apply Ingress Whitelist Annotations + policies.kyverno.io/subject: Ingress + policies.kyverno.io/description: >- + This policy creates annotations on ingresses. When + the `external-dns.home.arpa/enabled` annotation is not + set it applies the nginx annotations for use with only + internal application access. +spec: + mutateExistingOnPolicyUpdate: true + generateExistingOnPolicyUpdate: true + rules: + - name: whitelist + match: + any: + - resources: + kinds: ["Ingress"] + exclude: + any: + - resources: + annotations: + external-dns.home.arpa/enabled: "true" + mutate: + patchStrategicMerge: + metadata: + annotations: + +(nginx.ingress.kubernetes.io/whitelist-source-range): |- + 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 diff --git a/cluster/apps/kube-tools/kyverno/policies/delete-cpu-limits.yaml b/cluster/apps/kube-tools/kyverno/policies/delete-cpu-limits.yaml new file mode 100644 index 000000000..85eb8cba9 --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/delete-cpu-limits.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: delete-cpu-limits + annotations: + policies.kyverno.io/title: Delete CPU limits + policies.kyverno.io/subject: Pod + policies.kyverno.io/description: >- + This policy deletes CPU limits from all Pods. +spec: + mutateExistingOnPolicyUpdate: true + generateExistingOnPolicyUpdate: true + rules: + - name: delete-cpu-limits + match: + any: + - resources: + kinds: ["Pod"] + exclude: + any: + - resources: + namespaces: + - calico-system + - tigera-operator + - resources: + kinds: ["Pod"] + selector: + matchLabels: + job-name: "*" + - resources: + kinds: ["Pod"] + selector: + matchLabels: + statefulset.kubernetes.io/pod-name: "*" + - resources: + annotations: + kyverno.io/ignore: "true" + mutate: + patchStrategicMerge: + spec: + initContainers: + - (name): "*" + resources: + limits: + cpu: null + containers: + - (name): "*" + resources: + limits: + cpu: null diff --git a/cluster/apps/kube-tools/kyverno/policies/kustomization.yaml b/cluster/apps/kube-tools/kyverno/policies/kustomization.yaml new file mode 100644 index 000000000..bab5ead7f --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - apply-ingress-auth-annotations.yaml + - apply-ingress-external-dns-annotations.yaml + - apply-ingress-whitelist-annotations.yaml + - delete-cpu-limits.yaml + - snapshot-cronjob-controller.yaml + - sync-postgres-secrets.yaml diff --git a/cluster/apps/kube-tools/kyverno/policies/snapshot-cronjob-controller.yaml b/cluster/apps/kube-tools/kyverno/policies/snapshot-cronjob-controller.yaml new file mode 100644 index 000000000..7d3e74c85 --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/snapshot-cronjob-controller.yaml @@ -0,0 +1,137 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: snapshot-cronjob-controller + annotations: + policies.kyverno.io/title: Snapshot CronJob controller + policies.kyverno.io/subject: PersistentVolumeClaim + policies.kyverno.io/description: | + This policy creates a Kopia snapshot CronJob for labeled PersistentVolumeClaims + + The following labels on PVCs with their respective labels are required for this to run: + - snapshot.home.arpa/enabled + - app.kubernetes.io/name + - app.kubernetes.io/instance + + An optional label of "snapshot.home.arpa/ignoreAffinity" may be set on the PVC + if the pod is guaranteed to not run during the time of this jobs execution +spec: + generateExistingOnPolicyUpdate: true + mutateExistingOnPolicyUpdate: true + rules: + - name: create-snapshot-cronjob + match: + any: + - resources: + kinds: + - PersistentVolumeClaim + selector: + matchLabels: + snapshot.home.arpa/enabled: "true" + app.kubernetes.io/name: "*" + app.kubernetes.io/instance: "*" + context: + - name: appName + variable: + jmesPath: "request.object.metadata.labels.\"app.kubernetes.io/name\"" + - name: claimName + variable: + jmesPath: "request.object.metadata.name" + - name: namespace + variable: + jmesPath: "request.object.metadata.namespace" + - name: nodeAffinity + variable: + value: + ignored: "{{ (request.object.metadata.labels.\"snapshot.home.arpa/ignoreAffinity\" || 'false') == 'false' }}" + labels: + - key: app.kubernetes.io/name + operator: "In" + values: + - "{{ request.object.metadata.labels.\"app.kubernetes.io/name\" }}" + - key: app.kubernetes.io/instance + operator: "In" + values: + - "{{ request.object.metadata.labels.\"app.kubernetes.io/instance\" }}" + generate: + synchronize: true + apiVersion: batch/v1 + kind: CronJob + name: "{{ appName }}-{{ claimName }}-snapshot" + namespace: "{{ request.object.metadata.namespace }}" + data: + metadata: + labels: + app.kubernetes.io/name: "{{ request.object.metadata.labels.\"app.kubernetes.io/name\" }}" + app.kubernetes.io/instance: "{{ request.object.metadata.labels.\"app.kubernetes.io/instance\" }}" + ownerReferences: + - apiVersion: "{{ request.object.apiVersion }}" + kind: "{{ request.object.kind }}" + name: "{{ request.object.metadata.name }}" + uid: "{{ request.object.metadata.uid }}" + spec: + schedule: "0 3 * * *" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + # Keep at least one job in completed state in accordance to the schedule + ttlSecondsAfterFinished: 86400 + template: + spec: + automountServiceAccountToken: false + restartPolicy: OnFailure + # Stagger jobs to run randomly within X seconds to avoid bringing down all apps at once + initContainers: + - name: wait + image: ghcr.io/onedr0p/kopia:0.11.3@sha256:72406602c99357951cb7284abbf88699081d60f6cffd22baddd8a6a2afe919f5 + command: ["/scripts/sleep.sh"] + args: ["1", "900"] + containers: + - name: snapshot + image: ghcr.io/onedr0p/kopia:0.11.3@sha256:72406602c99357951cb7284abbf88699081d60f6cffd22baddd8a6a2afe919f5 + env: + - name: KOPIA_CACHE_DIRECTORY + value: /snapshots/{{ namespace }}/{{ appName }}/{{ claimName }}/cache + - name: KOPIA_LOG_DIR + value: /snapshots/{{ namespace }}/{{ appName }}/{{ claimName }}/logs + - name: KOPIA_PASSWORD + value: "none" + command: + - /bin/bash + - -c + - |- + printf "\e[1;32m%-6s\e[m\n" "[01/10] Create repo ..." && [[ ! -f /snapshots/kopia.repository.f ]] && kopia repository create filesystem --path=/snapshots + printf "\e[1;32m%-6s\e[m\n" "[02/10] Connect to repo ..." && kopia repo connect filesystem --path=/snapshots --override-hostname=cluster --override-username=root + printf "\e[1;32m%-6s\e[m\n" "[03/10] Set policies ..." && kopia policy set /data/{{ namespace }}/{{ appName }}/{{ claimName }} --compression=zstd --keep-latest 14 --keep-hourly 0 --keep-daily 7 --keep-weekly 2 --keep-monthly 0 --keep-annual 0 + printf "\e[1;32m%-6s\e[m\n" "[04/10] Freeze {{ claimName }} ..." && fsfreeze -f /data/{{ namespace }}/{{ appName }}/{{ claimName }} + printf "\e[1;32m%-6s\e[m\n" "[05/10] Snapshot {{ claimName }} ..." && kopia snap create /data/{{ namespace }}/{{ appName }}/{{ claimName }} + printf "\e[1;32m%-6s\e[m\n" "[06/10] Unfreeze {{ claimName }} ..." && fsfreeze -u /data/{{ namespace }}/{{ appName }}/{{ claimName }} + printf "\e[1;32m%-6s\e[m\n" "[07/10] List snapshots ..." && kopia snap list /data/{{ namespace }}/{{ appName }}/{{ claimName }} + printf "\e[1;32m%-6s\e[m\n" "[08/10] Show stats ..." && kopia content stats + printf "\e[1;32m%-6s\e[m\n" "[09/10] Show maintenance info ..." && kopia maintenance info + printf "\e[1;32m%-6s\e[m\n" "[10/10] Disconnect from repo ..." && kopia repo disconnect + volumeMounts: + - name: data + mountPath: "/data/{{ namespace }}/{{ appName }}/{{ claimName }}" + - name: snapshots + mountPath: /snapshots + securityContext: + privileged: true + volumes: + - name: data + persistentVolumeClaim: + claimName: "{{ claimName }}" + - name: snapshots + nfs: + server: "expanse.${SECRET_PRIVATE_DOMAIN}" + path: /eros/Apps/Kopia + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: "{{ nodeAffinity.ignored && [] || nodeAffinity.labels }}" diff --git a/cluster/apps/kube-tools/kyverno/policies/sync-postgres-secrets.yaml b/cluster/apps/kube-tools/kyverno/policies/sync-postgres-secrets.yaml new file mode 100644 index 000000000..5b209b27f --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/policies/sync-postgres-secrets.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: sync-postgres-secrets + annotations: + policies.kyverno.io/title: Sync Postgres Secrets + policies.kyverno.io/subject: Secret + policies.kyverno.io/description: >- + This policy will copy a secret called `postgres-superuser` which + exists in the `database` namespace to new namespaces when they are + created. It will also push updates to the copied Secrets should the + source secret be changed. +spec: + mutateExistingOnPolicyUpdate: true + generateExistingOnPolicyUpdate: true + rules: + - name: sync-postgres-superuser-secret + match: + resources: + kinds: ["Namespace"] + exclude: + resources: + namespaces: ["default"] + generate: + apiVersion: v1 + kind: Secret + name: postgres-superuser + namespace: "{{request.object.metadata.name}}" + synchronize: true + clone: + namespace: default + name: postgres-superuser diff --git a/cluster/apps/kube-tools/kyverno/rbac.yaml b/cluster/apps/kube-tools/kyverno/rbac.yaml new file mode 100644 index 000000000..298701b56 --- /dev/null +++ b/cluster/apps/kube-tools/kyverno/rbac.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kyverno:admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +subjects: + - kind: ServiceAccount + name: kyverno + namespace: kyverno diff --git a/cluster/apps/kube-tools/metrics-server/helm-release.yaml b/cluster/apps/kube-tools/metrics-server/helm-release.yaml new file mode 100644 index 000000000..41ba81cd1 --- /dev/null +++ b/cluster/apps/kube-tools/metrics-server/helm-release.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: metrics-server + namespace: default +spec: + interval: 15m + chart: + spec: + chart: metrics-server + version: 3.8.2 + sourceRef: + kind: HelmRepository + name: metrics-server-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + args: + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + metrics: + enabled: true + serviceMonitor: + enabled: true diff --git a/cluster/core/kube-system/descheduler/kustomization.yaml b/cluster/apps/kube-tools/metrics-server/kustomization.yaml similarity index 96% rename from cluster/core/kube-system/descheduler/kustomization.yaml rename to cluster/apps/kube-tools/metrics-server/kustomization.yaml index 34a8531ce..2fa2de20c 100644 --- a/cluster/core/kube-system/descheduler/kustomization.yaml +++ b/cluster/apps/kube-tools/metrics-server/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/kube-system/node-feature-discovery/helm-release.yaml b/cluster/apps/kube-tools/node-feature-discovery/helm-release.yaml similarity index 88% rename from cluster/core/kube-system/node-feature-discovery/helm-release.yaml rename to cluster/apps/kube-tools/node-feature-discovery/helm-release.yaml index fd7dbe4f0..954c5d119 100644 --- a/cluster/core/kube-system/node-feature-discovery/helm-release.yaml +++ b/cluster/apps/kube-tools/node-feature-discovery/helm-release.yaml @@ -3,29 +3,32 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: name: node-feature-discovery - namespace: kube-system + namespace: default spec: - interval: 5m + interval: 15m chart: spec: - # renovate: registryUrl=https://kubernetes-sigs.github.io/node-feature-discovery/charts chart: node-feature-discovery version: 0.11.2 sourceRef: kind: HelmRepository name: node-feature-discovery-charts namespace: flux-system - interval: 5m install: createNamespace: true remediation: retries: 5 + upgrade: + remediation: + retries: 5 values: nodeFeatureRule: createCRD: false worker: annotations: configmap.reloader.stakater.com/reload: node-feature-discovery-worker-conf + nodeSelector: + node-role.kubernetes.io/worker: "true" config: core: sources: @@ -42,9 +45,9 @@ spec: - "fe" - "ff" deviceLabelFields: - - "class" - - "vendor" - - "device" + - class + - vendor + - device custom: - name: "zwave" matchOn: diff --git a/cluster/apps/kube-tools/node-feature-discovery/kustomization.yaml b/cluster/apps/kube-tools/node-feature-discovery/kustomization.yaml new file mode 100644 index 000000000..4e694c9e7 --- /dev/null +++ b/cluster/apps/kube-tools/node-feature-discovery/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - github.com/kubernetes-sigs/node-feature-discovery//deployment/base/nfd-crds?ref=v0.11.2 + - helm-release.yaml diff --git a/cluster/apps/kube-tools/reloader/helm-release.yaml b/cluster/apps/kube-tools/reloader/helm-release.yaml new file mode 100644 index 000000000..6f1c6ef3a --- /dev/null +++ b/cluster/apps/kube-tools/reloader/helm-release.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &name reloader + namespace: &namespace default +spec: + interval: 15m + chart: + spec: + chart: *name + version: v0.0.118 + sourceRef: + kind: HelmRepository + name: stakater-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + fullnameOverride: *name + reloader: + reloadStrategy: annotations + podMonitor: + enabled: true + namespace: *namespace diff --git a/cluster/apps/kube-tools/reloader/kustomization.yaml b/cluster/apps/kube-tools/reloader/kustomization.yaml new file mode 100644 index 000000000..2fa2de20c --- /dev/null +++ b/cluster/apps/kube-tools/reloader/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-release.yaml diff --git a/cluster/apps/kube-tools/system-upgrade/kustomization.yaml b/cluster/apps/kube-tools/system-upgrade/kustomization.yaml new file mode 100644 index 000000000..d8d3d0c17 --- /dev/null +++ b/cluster/apps/kube-tools/system-upgrade/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: system-upgrade +resources: + # renovate: datasource=docker image=rancher/system-upgrade-controller + - https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/crd.yaml + - system-upgrade-controller diff --git a/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/kustomization.yaml b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/kustomization.yaml new file mode 100644 index 000000000..531d6c97d --- /dev/null +++ b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/kustomization.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - github.com/rancher/system-upgrade-controller?ref=v0.9.1 + - plans +images: + - name: rancher/system-upgrade-controller + newTag: v0.9.1 +patchesStrategicMerge: + # Delete namespace resource + - ./system-upgrade-patches.yaml + # Add labels + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: system-upgrade-controller + namespace: system-upgrade + labels: + app.kubernetes.io/name: system-upgrade-controller + app.kubernetes.io/instance: system-upgrade-controller diff --git a/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/agent.yaml b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/agent.yaml new file mode 100644 index 000000000..6abc0f991 --- /dev/null +++ b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/agent.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: k3s-agent + namespace: system-upgrade + labels: + k3s-upgrade: agent +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: "v1.24.4+k3s1" + serviceAccountName: system-upgrade + concurrency: 1 + nodeSelector: + matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist} + prepare: + image: rancher/k3s-upgrade + args: ["prepare", "k3s-server"] + upgrade: + image: rancher/k3s-upgrade diff --git a/cluster/core/system-upgrade/kustomization.yaml b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/kustomization.yaml similarity index 71% rename from cluster/core/system-upgrade/kustomization.yaml rename to cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/kustomization.yaml index e65b68e99..d2f0a0394 100644 --- a/cluster/core/system-upgrade/kustomization.yaml +++ b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/kustomization.yaml @@ -2,5 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - controller - - plans + - server.yaml + - agent.yaml diff --git a/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/server.yaml b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/server.yaml new file mode 100644 index 000000000..969ff7edc --- /dev/null +++ b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/plans/server.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: k3s-server + namespace: system-upgrade + labels: + k3s-upgrade: server +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: "v1.24.4+k3s1" + serviceAccountName: system-upgrade + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: Exists} + upgrade: + image: rancher/k3s-upgrade diff --git a/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/system-upgrade-patches.yaml b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/system-upgrade-patches.yaml new file mode 100644 index 000000000..2161d7b0e --- /dev/null +++ b/cluster/apps/kube-tools/system-upgrade/system-upgrade-controller/system-upgrade-patches.yaml @@ -0,0 +1,9 @@ +--- +# Namespace should already exist +# Delete the system-upgrade namespace +# from the kustomization +$patch: delete +apiVersion: v1 +kind: Namespace +metadata: + name: system-upgrade diff --git a/cluster/apps/kube-tools/vpa/helm-release.yaml b/cluster/apps/kube-tools/vpa/helm-release.yaml new file mode 100644 index 000000000..59e4fd57a --- /dev/null +++ b/cluster/apps/kube-tools/vpa/helm-release.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: vpa + namespace: default +spec: + interval: 15m + chart: + spec: + chart: vpa + version: 1.4.0 + sourceRef: + kind: HelmRepository + name: fairwinds-charts + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 5 + upgrade: + remediation: + retries: 5 + values: + recommender: + enabled: true + extraArgs: + storage: prometheus + prometheus-address: |- + http://thanos-query.monitoring.svc.cluster.local:9090 + updater: + enabled: false + admissionController: + enabled: false diff --git a/cluster/apps/kube-tools/vpa/kustomization.yaml b/cluster/apps/kube-tools/vpa/kustomization.yaml new file mode 100644 index 000000000..2fa2de20c --- /dev/null +++ b/cluster/apps/kube-tools/vpa/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-release.yaml diff --git a/cluster/apps/kustomization.yaml b/cluster/apps/kustomization.yaml index 89a4988d3..d1be515c1 100644 --- a/cluster/apps/kustomization.yaml +++ b/cluster/apps/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - namespaces.yaml - authentication - data - databases @@ -9,6 +10,7 @@ resources: - downloaders - home-automation - kasten-io + - kube-tools - media - monitoring - networking diff --git a/cluster/apps/monitoring/kustomization.yaml b/cluster/apps/monitoring/kustomization.yaml index 3dd92412f..6e00df5e5 100644 --- a/cluster/apps/monitoring/kustomization.yaml +++ b/cluster/apps/monitoring/kustomization.yaml @@ -1,7 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - namespace.yaml - blackbox-exporter - grafana - healthchecks diff --git a/cluster/apps/monitoring/namespace.yaml b/cluster/apps/monitoring/namespace.yaml deleted file mode 100644 index ff7ae1b93..000000000 --- a/cluster/apps/monitoring/namespace.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: monitoring diff --git a/cluster/apps/namespaces.yaml b/cluster/apps/namespaces.yaml new file mode 100644 index 000000000..45bab7f25 --- /dev/null +++ b/cluster/apps/namespaces.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: calico-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: default + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: flux-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kyverno + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: system-upgrade + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tigera-operator + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + goldilocks.fairwinds.com/enabled: "true" diff --git a/cluster/charts/kubernetes-sigs-descheduler-charts.yaml b/cluster/charts/descheduler-charts.yaml similarity index 56% rename from cluster/charts/kubernetes-sigs-descheduler-charts.yaml rename to cluster/charts/descheduler-charts.yaml index 57ffa9b1c..06788a095 100644 --- a/cluster/charts/kubernetes-sigs-descheduler-charts.yaml +++ b/cluster/charts/descheduler-charts.yaml @@ -1,10 +1,9 @@ --- -apiVersion: source.toolkit.fluxcd.io/v1beta1 +apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: kubernetes-sigs-descheduler-charts + name: descheduler-charts namespace: flux-system spec: interval: 1h url: https://kubernetes-sigs.github.io/descheduler - timeout: 3m diff --git a/cluster/charts/fairwinds-charts.yaml b/cluster/charts/fairwinds-charts.yaml new file mode 100644 index 000000000..2daab4251 --- /dev/null +++ b/cluster/charts/fairwinds-charts.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: fairwinds-charts + namespace: flux-system +spec: + interval: 1h + url: https://charts.fairwinds.com/stable diff --git a/cluster/charts/kustomization.yaml b/cluster/charts/kustomization.yaml index 6f834f4cc..eb0276673 100644 --- a/cluster/charts/kustomization.yaml +++ b/cluster/charts/kustomization.yaml @@ -5,8 +5,10 @@ resources: - bitnami-charts.yaml - bjw-s-charts.yaml - cert-manager-webhook-ovh.yaml + - descheduler-charts.yaml - emxq-charts.yaml - external-dns-charts.yaml + - fairwinds-charts.yaml - gitea-charts.yaml - grafana-charts.yaml - influxdata-charts.yaml @@ -15,7 +17,8 @@ resources: - k8s-at-home.yaml - k8s-gateway-charts.yaml - kasten-charts.yaml - - kubernetes-sigs-descheduler-charts.yaml + - kyverno-charts.yaml + - metrics-server-charts.yaml - node-feature-discovery.yaml - prometheus-community-charts.yaml - rook-ceph-charts.yaml diff --git a/cluster/charts/kyverno-charts.yaml b/cluster/charts/kyverno-charts.yaml new file mode 100644 index 000000000..7257a920f --- /dev/null +++ b/cluster/charts/kyverno-charts.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: kyverno-charts + namespace: flux-system +spec: + interval: 1h + url: https://kyverno.github.io/kyverno/ diff --git a/cluster/charts/metrics-server-charts.yaml b/cluster/charts/metrics-server-charts.yaml new file mode 100644 index 000000000..50c80c1b5 --- /dev/null +++ b/cluster/charts/metrics-server-charts.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: metrics-server-charts + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes-sigs.github.io/metrics-server diff --git a/cluster/core/flux-system/webhook/github/ingress.yaml b/cluster/core/flux-system/webhook/github/ingress.yaml index 3ee920c04..57aece74f 100644 --- a/cluster/core/flux-system/webhook/github/ingress.yaml +++ b/cluster/core/flux-system/webhook/github/ingress.yaml @@ -5,8 +5,7 @@ metadata: name: webhook-receiver namespace: flux-system annotations: - external-dns.alpha.kubernetes.io/target: "services.${SECRET_DOMAIN}." - external-dns/is-public: "true" + external-dns.home.arpa/enabled: "true" spec: ingressClassName: "nginx" rules: diff --git a/cluster/core/kube-system/descheduler/helm-release.yaml b/cluster/core/kube-system/descheduler/helm-release.yaml deleted file mode 100644 index e69ed97b2..000000000 --- a/cluster/core/kube-system/descheduler/helm-release.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: descheduler - namespace: kube-system -spec: - interval: 5m - chart: - spec: - # renovate: registryUrl=https://kubernetes-sigs.github.io/descheduler - chart: descheduler-helm-chart - version: 0.19.1 - sourceRef: - kind: HelmRepository - name: kubernetes-sigs-descheduler-charts - namespace: flux-system - interval: 5m - values: - #schedule: "*/15 * * * *" - podAnnotations: - botkube.io/disable: "true" - deschedulerPolicy: - strategies: - RemoveDuplicates: - enabled: false - RemovePodsViolatingNodeAffinity: - enabled: true - params: - nodeAffinityType: - - requiredDuringSchedulingIgnoredDuringExecution - RemovePodsViolatingInterPodAntiAffinity: - enabled: false - LowNodeUtilization: - enabled: false diff --git a/cluster/core/kube-system/intel-gpu-plugin/helm-release.yaml b/cluster/core/kube-system/intel-gpu-plugin/helm-release.yaml deleted file mode 100644 index 069f36d9e..000000000 --- a/cluster/core/kube-system/intel-gpu-plugin/helm-release.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: intel-gpu-plugin - namespace: kube-system -spec: - interval: 5m - chart: - spec: - # renovate: registryUrl=https://k8s-at-home.com/charts/ - chart: intel-gpu-plugin - version: 4.4.2 - sourceRef: - kind: HelmRepository - name: k8s-at-home-charts - namespace: flux-system - interval: 5m - values: - image: - repository: ghcr.io/k8s-at-home/intel-gpu-plugin - tag: v0.22.0 - pullPolicy: IfNotPresent - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: feature.node.kubernetes.io/custom-intel-gpu - operator: In - values: - - "true" diff --git a/cluster/core/kube-system/namespace.yaml b/cluster/core/kube-system/namespace.yaml deleted file mode 100644 index 5988ffb9b..000000000 --- a/cluster/core/kube-system/namespace.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system diff --git a/cluster/core/kube-system/reloader/helm-release.yaml b/cluster/core/kube-system/reloader/helm-release.yaml deleted file mode 100644 index 9de8aeaed..000000000 --- a/cluster/core/kube-system/reloader/helm-release.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: reloader - namespace: kube-system -spec: - interval: 5m - chart: - spec: - # renovate: registryUrl=https://stakater.github.io/stakater-charts - chart: reloader - version: v0.0.118 - sourceRef: - kind: HelmRepository - name: stakater-charts - namespace: flux-system - interval: 5m - values: - nameOverride: reloader - fullnameOverride: reloader - reloader: - podMonitor: - enabled: true - namespace: kube-system diff --git a/cluster/core/kustomization.yaml b/cluster/core/kustomization.yaml index ab5ce0deb..d27b5e82d 100644 --- a/cluster/core/kustomization.yaml +++ b/cluster/core/kustomization.yaml @@ -4,7 +4,5 @@ resources: - cert-manager - flux-system - kasten-io - - kube-system - rook-ceph - - system-upgrade - storageclasses.yaml diff --git a/cluster/core/system-upgrade/controller/kustomization.yaml b/cluster/core/system-upgrade/controller/kustomization.yaml deleted file mode 100644 index fa9266261..000000000 --- a/cluster/core/system-upgrade/controller/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - github.com/rancher/system-upgrade-controller?ref=v0.9.1 -images: - - name: rancher/system-upgrade-controller - newTag: v0.9.1 diff --git a/cluster/core/system-upgrade/plans/kustomization.yaml b/cluster/core/system-upgrade/plans/kustomization.yaml deleted file mode 100644 index 457c088e9..000000000 --- a/cluster/core/system-upgrade/plans/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - server-plan.yaml - - worker-plan.yaml diff --git a/cluster/core/system-upgrade/plans/server-plan.yaml b/cluster/core/system-upgrade/plans/server-plan.yaml deleted file mode 100644 index 73cbb940f..000000000 --- a/cluster/core/system-upgrade/plans/server-plan.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: k3s-server - namespace: system-upgrade - labels: - k3s-upgrade: server -spec: - concurrency: 1 # Batch size (roughly maps to maximum number of unschedulable nodes) - channel: https://update.k3s.io/v1-release/channels/v1.24 - nodeSelector: - matchExpressions: - - { key: k3s-upgrade, operator: Exists } - - { key: k3s-upgrade, operator: NotIn, values: ["disabled", "false"] } - - { key: k3os.io/mode, operator: DoesNotExist } - - { key: node-role.kubernetes.io/master, operator: Exists } - tolerations: - - key: "node-role.kubernetes.io/control-plane" - operator: "Exists" - effect: "NoSchedule" - serviceAccountName: system-upgrade - cordon: true - upgrade: - image: rancher/k3s-upgrade diff --git a/cluster/core/system-upgrade/plans/worker-plan.yaml b/cluster/core/system-upgrade/plans/worker-plan.yaml deleted file mode 100644 index edd62ac3a..000000000 --- a/cluster/core/system-upgrade/plans/worker-plan.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- -# -# Worker plan -# -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: k3s-worker - namespace: system-upgrade - labels: - k3s-upgrade: worker -spec: - concurrency: 1 - channel: https://update.k3s.io/v1-release/channels/v1.24 - nodeSelector: - matchExpressions: - #- key: k3s-upgrade - # operator: Exists - - key: k3s-upgrade - operator: NotIn - values: - - "disabled" - - "false" - - key: kubernetes.io/hostname - operator: Exists - - key: k3os.io/mode - operator: DoesNotExist - - key: node-role.kubernetes.io/control-plane - operator: NotIn - values: - - "true" - serviceAccountName: system-upgrade - tolerations: - - key: kubernetes.io/arch - effect: NoSchedule - operator: Equal - value: amd64 - - key: kubernetes.io/arch - effect: NoSchedule - operator: Equal - value: arm64 - - key: kubernetes.io/arch - effect: NoSchedule - operator: Equal - value: arm - - key: arm - operator: Exists - prepare: - image: rancher/k3s-upgrade - args: - - "prepare" - - "k3s-server" - upgrade: - image: rancher/k3s-upgrade