From 92f83b38f0170c87a9e3356d5bcf08115efcacc6 Mon Sep 17 00:00:00 2001 From: auricom Date: Sat, 5 Dec 2020 09:15:39 +0100 Subject: [PATCH] add github actions split goldilocks yaml files fix github actions fix github actions fix github actions fix github actions fix github actions fix github actions fix github actions fix github actions fix github actions fix github actions Update Flux components to v0.4.3 [ci-skip] Signed-off-by: GitHub fix github actions fix github actions fix github actions --- .envrc | 1 + .github/renovate.json5 | 95 +++++++++++++++++++ .../renovate-helmrelease-schedule.yaml | 37 ++++++++ .github/workflows/update-flux.yaml | 43 +++++++++ .gitignore | 5 +- cluster/flux-system/gotk-components.yaml | 9 +- cluster/kube-system/ingress-nginx.yaml | 22 ++--- cluster/monitoring/goldilocks-crds.yaml | 94 ++++++++++++++++++ cluster/monitoring/goldilocks.yaml | 94 ------------------ scripts/create-helmrelease-annotations.sh | 39 ++++++++ 10 files changed, 332 insertions(+), 107 deletions(-) create mode 100644 .envrc create mode 100644 .github/renovate.json5 create mode 100644 .github/workflows/renovate-helmrelease-schedule.yaml create mode 100644 .github/workflows/update-flux.yaml create mode 100644 cluster/monitoring/goldilocks-crds.yaml create mode 100644 scripts/create-helmrelease-annotations.sh diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6762e217e --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +export KUBECONFIG=$(expand_path ./kubeconfig) \ No newline at end of file diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 000000000..838b109b7 --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,95 @@ +{ + "enabled": true, + "timezone": "Europe/Paris", + "dependencyDashboard": true, + "dependencyDashboardTitle": "Renovate Dashboard", + "commitMessageSuffix": "[ci-skip]", + // do not notify on closed, unmerged PRs + "suppressNotifications": ["prIgnoreNotification"], + // do not rebase PRs + "rebaseWhen": "conflicted", + "assignees": ["@auricom"], + "helm-values": { + "fileMatch": ["cluster/.+\\.yaml$"] + }, + "regexManagers": [ + { + "fileMatch": ["cluster/.+\\.yaml$"], + "matchStrings": [ + "registryUrl=(?.*?)\n *chart: (?.*?)\n *version: (?.*)\n" + ], + "datasourceTemplate": "helm" + } + ], + "packageRules": [ + // labels + { + "datasources": ["docker"], + "updateTypes": ["major"], + "labels": ["renovate/image-release", "dependency/major"] + }, + { + "datasources": ["docker"], + "updateTypes": ["minor"], + "labels": ["renovate/image-release", "dependency/minor"] + }, + { + "datasources": ["docker"], + "updateTypes": ["patch"], + "labels": ["renovate/image-release", "dependency/patch"] + }, + { + "datasources": ["helm"], + "updateTypes": ["major"], + "labels": ["renovate/helm-release", "dependency/major"] + }, + { + "datasources": ["helm"], + "updateTypes": ["minor"], + "labels": ["renovate/helm-release", "dependency/minor"] + }, + { + "datasources": ["helm"], + "updateTypes": ["patch"], + "labels": ["renovate/helm-release", "dependency/patch"] + }, + // interesting image tags + { + "datasources": ["docker"], + "versionScheme": "regex:^(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?.*)$", + "packageNames": ["blakeblackshear/frigate"] + }, + // pin versions + //{ + // "datasources": ["docker"], + // "allowedVersions": "<4.3.0", + // "packageNames": ["k8sathome/qbittorrent"] + //}, + // versioning + // { + // "datasources": ["docker"], + // "versioning": "nuget", + // "packageNames": [ + // "k8sathome/radarr", + // "k8sathome/sonarr", + // "k8sathome/lidarr", + // "k8sathome/bazarr" + // ] + // }, + { + "datasources": ["docker"], + "versioning": "loose", + "packageNames": ["plexinc/pms-docker"] + }, + // auto-updates + { + "datasources": ["docker"], + "automerge": true, + "requiredStatusChecks": null, + "updateTypes": ["minor", "patch"], + "packageNames": [ + "k8sathome/jackett" + ] + } + ] +} \ No newline at end of file diff --git a/.github/workflows/renovate-helmrelease-schedule.yaml b/.github/workflows/renovate-helmrelease-schedule.yaml new file mode 100644 index 000000000..46bfe3353 --- /dev/null +++ b/.github/workflows/renovate-helmrelease-schedule.yaml @@ -0,0 +1,37 @@ +name: renovate-helmrelease-schedule + +on: + workflow_dispatch: + schedule: + - cron: "15 * * * *" + +jobs: + renovate-helmrelease: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Install yq + run: | + mkdir -p "${GITHUB_WORKSPACE}/bin" + echo "${GITHUB_WORKSPACE}/bin" >> $GITHUB_PATH + curl -sL -o "${GITHUB_WORKSPACE}/bin/yq" https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 + chmod +x "${GITHUB_WORKSPACE}/bin/yq" + - name: Create HelmRelease annotations + run: ./hack/create-helmrelease-annotations.sh + + - name: Create pull request for HelmRelease annotations + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: helm-release-annotations + delete-branch: true + title: "Update HelmRelease annotations for Renovate [ci-skip]" + signoff: true + assignees: "auricom" + commit-message: "Update HelmRelease annotations for Renovate [ci-skip]" + body: | + Add comment annotations to HelmReleases inorder for Renovate to pick up new versions of Helm charts + labels: renovate/helm-release-annotations \ No newline at end of file diff --git a/.github/workflows/update-flux.yaml b/.github/workflows/update-flux.yaml new file mode 100644 index 000000000..12ecb1aad --- /dev/null +++ b/.github/workflows/update-flux.yaml @@ -0,0 +1,43 @@ +name: update-flux + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +jobs: + components: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Setup Flux CLI + uses: fluxcd/flux2/action@main + + - name: Setup Kubernetes + uses: engineerd/setup-kind@v0.5.0 + + - name: Check for updates + id: update + run: | + UGLY_VERSION="$(flux -v)" + VERSION="v${UGLY_VERSION#*flux version }" + flux install \ + --network-policy=false \ + --export > ./cluster/flux-system/gotk-components.yaml + + echo "::set-output name=flux_version::$VERSION" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: flux/update-${{ steps.update.outputs.flux_version }} + delete-branch: true + commit-message: Update Flux components to ${{ steps.update.outputs.flux_version }} [ci-skip] + title: Update Flux components to ${{ steps.update.outputs.flux_version }} [ci-skip] + signoff: true + assignees: "auricom" + body: | + Release notes: https://github.com/fluxcd/flux2/releases/tag/${{ steps.update.outputs.flux_version }} + labels: flux/update \ No newline at end of file diff --git a/.gitignore b/.gitignore index ee7c8d4a6..597be6812 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ # Secrets -*.clear \ No newline at end of file +*.clear +# Flux +flux +bin/ \ No newline at end of file diff --git a/cluster/flux-system/gotk-components.yaml b/cluster/flux-system/gotk-components.yaml index 0d83cf867..30ece3a77 100755 --- a/cluster/flux-system/gotk-components.yaml +++ b/cluster/flux-system/gotk-components.yaml @@ -1,3 +1,6 @@ +--- +# GitOps Toolkit revision latest +# Components: source-controller,kustomize-controller,helm-controller,notification-controller apiVersion: v1 kind: Namespace metadata: @@ -1389,6 +1392,9 @@ spec: install: description: Install holds the configuration for Helm install actions for this HelmRelease. properties: + createNamespace: + description: CreateNamespace tells the Helm install action to create the HelmReleaseSpec.TargetNamespace if it does not exist yet. On uninstall, the namespace will not be garbage collected. + type: boolean disableHooks: description: DisableHooks prevents hooks from running during the Helm install action. type: boolean @@ -1714,7 +1720,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/fluxcd/helm-controller:v0.4.0 + image: ghcr.io/fluxcd/helm-controller:v0.4.2 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -2329,3 +2335,4 @@ spec: volumes: - emptyDir: {} name: temp +--- diff --git a/cluster/kube-system/ingress-nginx.yaml b/cluster/kube-system/ingress-nginx.yaml index 875549a31..ebc0c6bc6 100644 --- a/cluster/kube-system/ingress-nginx.yaml +++ b/cluster/kube-system/ingress-nginx.yaml @@ -54,16 +54,16 @@ spec: values: - ingress-nginx-external topologyKey: "kubernetes.io/hostname" - defaultBackend: - enabled: true - image: - repository: registry.k3s.xpander.ovh/homelab/custom-error-pages - tag: 1.0.1 - resources: - requests: - memory: 50Mi - cpu: 25m - limits: - memory: 100Mi + #defaultBackend: + # enabled: true + # image: + # repository: registry.k3s.xpander.ovh/homelab/custom-error-pages + # tag: 1.0.1 + # resources: + # requests: + # memory: 50Mi + # cpu: 25m + # limits: + # memory: 100Mi tcp: 8086: monitoring/influxdb:8086 diff --git a/cluster/monitoring/goldilocks-crds.yaml b/cluster/monitoring/goldilocks-crds.yaml new file mode 100644 index 000000000..8b5a8563a --- /dev/null +++ b/cluster/monitoring/goldilocks-crds.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: verticalpodautoscalers.autoscaling.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/63797" +spec: + group: autoscaling.k8s.io + scope: Namespaced + names: + plural: verticalpodautoscalers + singular: verticalpodautoscaler + kind: VerticalPodAutoscaler + shortNames: + - vpa + version: v1beta1 + versions: + - name: v1beta1 + served: false + storage: false + - name: v1beta2 + served: true + storage: true + - name: v1 + served: true + storage: false + validation: + # openAPIV3Schema is the schema for validating custom objects. + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + targetRef: + type: object + updatePolicy: + type: object + properties: + updateMode: + type: string + resourcePolicy: + type: object + properties: + containerPolicies: + type: array + items: + type: object + properties: + containerName: + type: string + controlledValues: + type: string + enum: ["RequestsAndLimits", "RequestsOnly"] + mode: + type: string + enum: ["Auto", "Off"] + minAllowed: + type: object + maxAllowed: + type: object + controlledResources: + type: array + items: + type: string + enum: ["cpu", "memory"] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: verticalpodautoscalercheckpoints.autoscaling.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/63797" +spec: + group: autoscaling.k8s.io + scope: Namespaced + names: + plural: verticalpodautoscalercheckpoints + singular: verticalpodautoscalercheckpoint + kind: VerticalPodAutoscalerCheckpoint + shortNames: + - vpacheckpoint + version: v1beta1 + versions: + - name: v1beta1 + served: false + storage: false + - name: v1beta2 + served: true + storage: true + - name: v1 + served: true + storage: false \ No newline at end of file diff --git a/cluster/monitoring/goldilocks.yaml b/cluster/monitoring/goldilocks.yaml index a44d59ab1..fe667f77f 100644 --- a/cluster/monitoring/goldilocks.yaml +++ b/cluster/monitoring/goldilocks.yaml @@ -26,97 +26,3 @@ spec: valuesFrom: - kind: ConfigMap name: helmrelease-monitoring-goldilocks ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: verticalpodautoscalers.autoscaling.k8s.io - annotations: - "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/63797" -spec: - group: autoscaling.k8s.io - scope: Namespaced - names: - plural: verticalpodautoscalers - singular: verticalpodautoscaler - kind: VerticalPodAutoscaler - shortNames: - - vpa - version: v1beta1 - versions: - - name: v1beta1 - served: false - storage: false - - name: v1beta2 - served: true - storage: true - - name: v1 - served: true - storage: false - validation: - # openAPIV3Schema is the schema for validating custom objects. - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - targetRef: - type: object - updatePolicy: - type: object - properties: - updateMode: - type: string - resourcePolicy: - type: object - properties: - containerPolicies: - type: array - items: - type: object - properties: - containerName: - type: string - controlledValues: - type: string - enum: ["RequestsAndLimits", "RequestsOnly"] - mode: - type: string - enum: ["Auto", "Off"] - minAllowed: - type: object - maxAllowed: - type: object - controlledResources: - type: array - items: - type: string - enum: ["cpu", "memory"] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: verticalpodautoscalercheckpoints.autoscaling.k8s.io - annotations: - "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/63797" -spec: - group: autoscaling.k8s.io - scope: Namespaced - names: - plural: verticalpodautoscalercheckpoints - singular: verticalpodautoscalercheckpoint - kind: VerticalPodAutoscalerCheckpoint - shortNames: - - vpacheckpoint - version: v1beta1 - versions: - - name: v1beta1 - served: false - storage: false - - name: v1beta2 - served: true - storage: true - - name: v1 - served: true - storage: false \ No newline at end of file diff --git a/scripts/create-helmrelease-annotations.sh b/scripts/create-helmrelease-annotations.sh new file mode 100644 index 000000000..ccba802b8 --- /dev/null +++ b/scripts/create-helmrelease-annotations.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +shopt -s globstar + +# shellcheck disable=SC2155 +REPO_ROOT=$(git rev-parse --show-toplevel) +CLUSTER_ROOT="${REPO_ROOT}/cluster" +HELM_REPOSITORIES="${CLUSTER_ROOT}/flux-system/helm-chart-repositories" + +# Ensure yq exist +command -v yq >/dev/null 2>&1 || { + echo >&2 "yq is not installed. Aborting." + exit 1 +} + +for helm_release in "${CLUSTER_ROOT}"/**/*.yaml; do + # ignore flux-system namespace + # ignore wrong apiVersion + # ignore non HelmReleases + if [[ "${helm_release}" =~ "flux-system" + || $(yq r "${helm_release}" apiVersion) != "helm.toolkit.fluxcd.io/v2beta1" + || $(yq r "${helm_release}" kind) != "HelmRelease" ]]; then + continue + fi + + for helm_repository in "${HELM_REPOSITORIES}"/*.yaml; do + chart_name=$(yq r "${helm_repository}" metadata.name) + chart_url=$(yq r "${helm_repository}" spec.url) + + # only helmreleases where helm_release is related to chart_url + if [[ $(yq r "${helm_release}" spec.chart.spec.sourceRef.name) == "${chart_name}" ]]; then + # delete "renovate: registryUrl=" line + sed -i "/renovate: registryUrl=/d" "${helm_release}" + # insert "renovate: registryUrl=" line + sed -i "/.*chart: .*/i \ \ \ \ \ \ # renovate: registryUrl=${chart_url}" "${helm_release}" + echo "Annotated $(basename "${helm_release%.*}") with ${chart_name} for renovatebot..." + break + fi + done +done \ No newline at end of file