🚀 update github workflows

This commit is contained in:
auricom
2022-12-21 02:55:23 +01:00
parent 764ea76bda
commit 773f1ad63b
199 changed files with 543 additions and 752 deletions

View File

@@ -1,124 +0,0 @@
#!/usr/bin/env bash
# shellcheck source=/dev/null
source "$(dirname "${0}")/lib/functions.sh"
set -o errexit
set -o nounset
set -o pipefail
shopt -s lastpipe
show_help() {
cat << EOF
Usage: $(basename "$0") <options>
-h, --help Display help
-f, --file File to scan for container images
--nothing Enable nothing mode
EOF
}
main() {
local file=
local nothing=
parse_command_line "$@"
check "jo"
check "jq"
check "yq"
entry
}
parse_command_line() {
while :; do
case "${1:-}" in
-h|--help)
show_help
exit
;;
-f|--file)
if [[ -n "${2:-}" ]]; then
file="$2"
shift
else
echo "ERROR: '-f|--file' cannot be empty." >&2
show_help
exit 1
fi
;;
--nothing)
nothing=1
;;
*)
break
;;
esac
shift
done
if [[ -z "$file" ]]; then
echo "ERROR: '-f|--file' is required." >&2
show_help
exit 1
fi
if [[ -z "$nothing" ]]; then
nothing=0
fi
}
entry() {
# create new array to hold the images
images=()
# look in hydrated flux helm releases
chart_registry_url=$(chart_registry_url "${file}")
chart_name=$(yq eval-all .spec.chart.spec.chart "${file}" 2>/dev/null)
if [[ -n ${chart_registry_url} && -n "${chart_name}" && ! "${chart_name}" =~ "null" ]]; then
chart_version=$(yq eval .spec.chart.spec.version "${file}" 2>/dev/null)
chart_values=$(yq eval .spec.values "${file}" 2>/dev/null)
pushd "$(mktemp -d)" > /dev/null 2>&1
helm repo add main "${chart_registry_url}" > /dev/null 2>&1
helm pull "main/${chart_name}" --untar --version "${chart_version}"
resources=$(echo "${chart_values}" | helm template "${chart_name}" "${chart_name}" --version "${chart_version}" -f -)
popd > /dev/null 2>&1
images+=("$(echo "${resources}" | yq eval-all '.spec.template.spec.containers.[].image' -)")
helm repo remove main > /dev/null 2>&1
fi
# look in helm values
images+=("$(yq eval-all '[.. | select(has("repository")) | select(has("tag"))] | .[] | .repository + ":" + .tag' "${file}" 2>/dev/null)")
# look in kubernetes deployments, statefulsets and daemonsets
images+=("$(yq eval-all '.spec.template.spec.containers.[].image' "${file}" 2>/dev/null)")
# look in kubernetes pods
images+=("$(yq eval-all '.spec.containers.[].image' "${file}" 2>/dev/null)")
# look in kubernetes cronjobs
images+=("$(yq eval-all '.spec.jobTemplate.spec.template.spec.containers.[].image' "${file}" 2>/dev/null)")
# look in docker compose
images+=("$(yq eval-all '.services.*.image' "${file}" 2>/dev/null)")
# remove duplicate values xD
IFS=" " read -r -a images <<< "$(tr ' ' '\n' <<< "${images[@]}" | sort -u | tr '\n' ' ')"
# create new array to hold the parsed images
parsed_images=()
# loop thru the images removing any invalid items
for i in "${images[@]}"; do
# loop thru each image and split on new lines (for when yq finds multiple containers in the same file)
for b in ${i//\\n/ }; do
if [[ -z "${b}" || "${b}" == "null" || "${b}" == "---" ]]; then
continue
fi
parsed_images+=("${b}")
done
done
# check if parsed_images array has items
if (( ${#parsed_images[@]} )); then
# convert the bash array to json and wrap array in an containers object
jo -a "${parsed_images[@]}" | jq -c '{containers: [(.[])]}'
fi
}
main "$@"

View File

@@ -1,175 +0,0 @@
#!/usr/bin/env bash
# shellcheck source=/dev/null
source "$(dirname "${0}")/lib/functions.sh"
set -o errexit
set -o nounset
set -o pipefail
shopt -s lastpipe
show_help() {
cat << EOF
Usage: $(basename "$0") <options>
-h, --help Display help
--source-file Original helm release
--target-file New helm release
--remove-common-labels Remove common labels from manifests
EOF
}
main() {
local source_file=
local target_file=
local remove_common_labels=
parse_command_line "$@"
check "helm"
check "yq"
entry
}
parse_command_line() {
while :; do
case "${1:-}" in
-h|--help)
show_help
exit
;;
--source-file)
if [[ -n "${2:-}" ]]; then
source_file="$2"
shift
else
echo "ERROR: '--source-file' cannot be empty." >&2
show_help
exit 1
fi
;;
--target-file)
if [[ -n "${2:-}" ]]; then
target_file="$2"
shift
else
echo "ERROR: '--target-file' cannot be empty." >&2
show_help
exit 1
fi
;;
--remove-common-labels)
remove_common_labels=true
;;
*)
break
;;
esac
shift
done
if [[ -z "${source_file}" ]]; then
echo "ERROR: '--source-file' is required." >&2
show_help
exit 1
fi
if [[ $(yq eval .kind "${source_file}" 2>/dev/null) != "HelmRelease" ]]; then
echo "ERROR: '--source-file' is not a HelmRelease"
show_help
exit 1
fi
if [[ -z "${target_file}" ]]; then
echo "ERROR: '--target-file' is required." >&2
show_help
exit 1
fi
if [[ $(yq eval .kind "${target_file}" 2>/dev/null) != "HelmRelease" ]]; then
echo "ERROR: '--target-file' is not a HelmRelease"
show_help
exit 1
fi
if [[ -z "$remove_common_labels" ]]; then
remove_common_labels=false
fi
}
_resources() {
local chart_name=${1}
local chart_version=${2}
local chart_registry_url=${3}
local chart_values=${4}
local resources=
helm repo add main "${chart_registry_url}" > /dev/null 2>&1
pushd "$(mktemp -d)" > /dev/null 2>&1
helm pull "main/${chart_name}" --untar --version "${chart_version}"
resources=$(echo "${chart_values}" | helm template "${chart_name}" "${chart_name}" --version "${chart_version}" -f -)
if [[ "${remove_common_labels}" == "true" ]]; then
labels='.metadata.labels."helm.sh/chart"'
labels+=',.metadata.labels.chart'
labels+=',.metadata.labels."app.kubernetes.io/version"'
labels+=',.spec.template.metadata.labels."helm.sh/chart"'
labels+=',.spec.template.metadata.labels.chart'
labels+=',.spec.template.metadata.labels."app.kubernetes.io/version"'
echo "${resources}" | yq eval "del($labels)" -
else
echo "${resources}"
fi
popd > /dev/null 2>&1
helm repo remove main > /dev/null 2>&1
}
entry() {
local comments=
source_chart_name=$(chart_name "${source_file}")
source_chart_version=$(chart_version "${source_file}")
source_chart_registry_url=$(chart_registry_url "${source_file}")
source_chart_values=$(chart_values "${source_file}")
source_resources=$(_resources "${source_chart_name}" "${source_chart_version}" "${source_chart_registry_url}" "${source_chart_values}")
echo "${source_resources}" > /tmp/source_resources
target_chart_version=$(chart_version "${target_file}")
target_chart_name=$(chart_name "${target_file}")
target_chart_registry_url=$(chart_registry_url "${target_file}")
target_chart_values=$(chart_values "${target_file}")
target_resources=$(_resources "${target_chart_name}" "${target_chart_version}" "${target_chart_registry_url}" "${target_chart_values}")
echo "${target_resources}" > /tmp/target_resources
# Diff the files and always return true
diff -u /tmp/source_resources /tmp/target_resources > /tmp/diff || true
# Remove the filenames
sed -i -e '1,2d' /tmp/diff
# Store the comment in an array
comments=()
# shellcheck disable=SC2016
comments+=( "$(printf 'Path: `%s`' "${target_file}")" )
if [[ "${source_chart_name}" != "${target_chart_name}" ]]; then
# shellcheck disable=SC2016
comments+=( "$(printf 'Chart: `%s` -> `%s`' "${source_chart_name}" "${target_chart_name}")" )
fi
if [[ "${source_chart_version}" != "${target_chart_version}" ]]; then
# shellcheck disable=SC2016
comments+=( "$(printf 'Version: `%s` -> `%s`' "${source_chart_version}" "${target_chart_version}")" )
fi
if [[ "${source_chart_registry_url}" != "${target_chart_registry_url}" ]]; then
# shellcheck disable=SC2016
comments+=( "$(printf 'Registry URL: `%s` -> `%s`' "${source_chart_registry_url}" "${target_chart_registry_url}")" )
fi
comments+=( "$(printf '\n\n')" )
if [[ -f /tmp/diff && -s /tmp/diff ]]; then
# shellcheck disable=SC2016
comments+=( "$(printf '```diff\n%s\n```' "$(cat /tmp/diff)")" )
else
# shellcheck disable=SC2016
comments+=( "$(printf '```\nNo changes in detected in resources\n```')" )
fi
# Join the array with a new line and print it
printf "%s\n" "${comments[@]}"
}
main "$@"

159
.github/scripts/helmReleaseDiff.mjs vendored Executable file
View File

@@ -0,0 +1,159 @@
#!/usr/bin/env zx
$.verbose = false;
/**
* * helmReleaseDiff.mjs
* * Runs `helm template` with your Helm values and then runs `dyff` across Flux HelmRelease manifests
* @param --current-release The source Flux HelmRelease to compare against the target
* @param --incoming-release The target Flux HelmRelease to compare against the source
* @param --kubernetes-dir The directory containing your Flux manifests including the HelmRepository manifests
* * Limitations:
* * Does not work with multiple HelmRelease maninfests in the same YAML document
*/
const CurrentRelease = argv["current-release"];
const IncomingRelease = argv["incoming-release"];
const KubernetesDir = argv["kubernetes-dir"];
const dyff = await which("dyff");
const helm = await which("helm");
const kustomize = await which("kustomize");
async function helmRelease(releaseFile) {
const helmRelease = await fs.readFile(releaseFile, "utf8");
const doc = YAML.parseAllDocuments(helmRelease).map((item) => item.toJS());
const release = doc.filter(
(item) =>
item.apiVersion === "helm.toolkit.fluxcd.io/v2beta1" &&
item.kind === "HelmRelease"
);
return release[0];
}
async function helmRepositoryUrl(kubernetesDir, releaseName) {
const files = await globby([`${kubernetesDir}/**/*.yaml`]);
for await (const file of files) {
const contents = await fs.readFile(file, "utf8");
const doc = YAML.parseAllDocuments(contents).map((item) => item.toJS());
if (
"apiVersion" in doc[0] &&
doc[0].apiVersion === "source.toolkit.fluxcd.io/v1beta2" &&
"kind" in doc[0] &&
doc[0].kind === "HelmRepository" &&
"metadata" in doc[0] &&
"name" in doc[0].metadata &&
doc[0].metadata.name === releaseName
) {
return doc[0].spec.url;
}
}
}
async function kustomizeBuild(releaseBaseDir, releaseName) {
const build =
await $`${kustomize} build --load-restrictor=LoadRestrictionsNone ${releaseBaseDir}`;
const docs = YAML.parseAllDocuments(build.stdout).map((item) => item.toJS());
const release = docs.filter(
(item) =>
item.apiVersion === "helm.toolkit.fluxcd.io/v2beta1" &&
item.kind === "HelmRelease" &&
item.metadata.name === releaseName
);
return release[0];
}
async function helmRepoAdd(registryName, registryUrl) {
await $`${helm} repo add ${registryName} ${registryUrl}`;
}
async function helmTemplate(
releaseName,
registryName,
chartName,
chartVersion,
chartValues
) {
const values = new YAML.Document();
values.contents = chartValues;
const valuesFile = await $`mktemp`;
await fs.writeFile(valuesFile.stdout.trim(), values.toString());
const manifestsFile = await $`mktemp`;
const manifests =
await $`${helm} template --kube-version 1.26.0 --release-name ${releaseName} --include-crds=false ${registryName}/${chartName} --version ${chartVersion} --values ${valuesFile.stdout.trim()}`;
// Remove docs that are CustomResourceDefinition and keys which contain generated fields
let documents = YAML.parseAllDocuments(manifests.stdout.trim());
documents = documents.filter(
(doc) => doc.get("kind") !== "CustomResourceDefinition"
);
documents.forEach((doc) => {
const del = (path) => (doc.hasIn(path) ? doc.deleteIn(path) : false);
del(["metadata", "labels", "app.kubernetes.io/version"]);
del(["metadata", "labels", "chart"]);
del(["metadata", "labels", "helm.sh/chart"]);
del([
"spec",
"template",
"metadata",
"labels",
"app.kubernetes.io/version",
]);
del(["spec", "template", "metadata", "labels", "chart"]);
del(["spec", "template", "metadata", "labels", "helm.sh/chart"]);
});
await fs.writeFile(
manifestsFile.stdout.trim(),
documents.map((doc) => doc.toString({ directives: true })).join("\n")
);
return manifestsFile.stdout.trim();
}
// Generate current template from Helm values
const currentRelease = await helmRelease(CurrentRelease);
const currentBuild = await kustomizeBuild(
path.dirname(CurrentRelease),
currentRelease.metadata.name
);
const currentRepositoryUrl = await helmRepositoryUrl(
KubernetesDir,
currentBuild.spec.chart.spec.sourceRef.name
);
await helmRepoAdd(
currentBuild.spec.chart.spec.sourceRef.name,
currentRepositoryUrl
);
const currentManifests = await helmTemplate(
currentBuild.metadata.name,
currentBuild.spec.chart.spec.sourceRef.name,
currentBuild.spec.chart.spec.chart,
currentBuild.spec.chart.spec.version,
currentBuild.spec.values
);
// Generate incoming template from Helm values
const incomingRelease = await helmRelease(IncomingRelease);
const incomingBuild = await kustomizeBuild(
path.dirname(IncomingRelease),
incomingRelease.metadata.name
);
const incomingRepositoryUrl = await helmRepositoryUrl(
KubernetesDir,
incomingBuild.spec.chart.spec.sourceRef.name
);
await helmRepoAdd(
incomingBuild.spec.chart.spec.sourceRef.name,
incomingRepositoryUrl
);
const incomingManifests = await helmTemplate(
incomingBuild.metadata.name,
incomingBuild.spec.chart.spec.sourceRef.name,
incomingBuild.spec.chart.spec.chart,
incomingBuild.spec.chart.spec.version,
incomingBuild.spec.values
);
// Print diff using dyff
const diff =
await $`${dyff} --color=off --truecolor=off between --omit-header --ignore-order-changes --detect-kubernetes=true --output=human ${currentManifests} ${incomingManifests}`;
echo(diff.stdout.trim());

67
.github/scripts/kubeconform.sh vendored Normal file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/env bash
set -o errexit
KUBERNETES_DIR=$1
SCHEMA_DIR=$2
KUBE_VERSION="${3:-1.26.0}"
[[ -z "${KUBERNETES_DIR}" ]] && echo "Kubernetes location not specified" && exit 1
[[ -z "${SCHEMA_DIR}" ]] && echo "Schema location not specified" && exit 1
kustomize_args=("--load-restrictor=LoadRestrictionsNone")
kustomize_config="kustomization.yaml"
kubeconform_args=(
"-strict"
"-ignore-missing-schemas"
"-kubernetes-version"
"${KUBE_VERSION}"
"-skip"
"Secret"
"-schema-location"
"default"
"-schema-location"
"${SCHEMA_DIR}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
"-verbose"
)
echo "=== Validating standalone manifests in ${KUBERNETES_DIR}/flux ==="
find "${KUBERNETES_DIR}/flux" -maxdepth 1 -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
do
kubeconform "${kubeconform_args[@]}" "${file}"
if [[ ${PIPESTATUS[0]} != 0 ]]; then
exit 1
fi
done
echo "=== Validating kustomizations in ${KUBERNETES_DIR}/flux ==="
find "${KUBERNETES_DIR}/flux" -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
do
echo "=== Validating kustomizations in ${file/%$kustomize_config} ==="
kustomize build "${file/%$kustomize_config}" "${kustomize_args[@]}" | \
kubeconform "${kubeconform_args[@]}"
if [[ ${PIPESTATUS[0]} != 0 ]]; then
exit 1
fi
done
echo "=== Validating kustomizations in ${KUBERNETES_DIR}/base ==="
find "${KUBERNETES_DIR}/base" -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
do
echo "=== Validating kustomizations in ${file/%$kustomize_config} ==="
kustomize build "${file/%$kustomize_config}" "${kustomize_args[@]}" | \
kubeconform "${kubeconform_args[@]}"
if [[ ${PIPESTATUS[0]} != 0 ]]; then
exit 1
fi
done
echo "=== Validating kustomizations in ${KUBERNETES_DIR}/cluster-0 ==="
find "${KUBERNETES_DIR}/cluster-0" -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
do
echo "=== Validating kustomizations in ${file/%$kustomize_config} ==="
kustomize build "${file/%$kustomize_config}" "${kustomize_args[@]}" | \
kubeconform "${kubeconform_args[@]}"
if [[ ${PIPESTATUS[0]} != 0 ]]; then
exit 1
fi
done

View File

@@ -1,47 +0,0 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
shopt -s lastpipe
check() {
command -v "${1}" >/dev/null 2>&1 || {
echo >&2 "ERROR: ${1} is not installed or not found in \$PATH" >&2
exit 1
}
}
chart_registry_url() {
local helm_release=
local chart_id=
helm_release="${1}"
chart_id=$(yq eval .spec.chart.spec.sourceRef.name "${helm_release}" 2>/dev/null)
# Discover all HelmRepository
find . -iname '*-charts.yaml' -type f -print0 | while IFS= read -r -d '' file; do
# Skip non HelmRepository
[[ $(yq eval .kind "${file}" 2>/dev/null) != "HelmRepository" ]] && continue
# Skip unrelated HelmRepository
[[ "${chart_id}" != $(yq eval .metadata.name "${file}" 2>/dev/null) ]] && continue
yq eval .spec.url "${file}"
break
done
}
chart_name() {
local helm_release=
helm_release="${1}"
yq eval .spec.chart.spec.chart "${helm_release}" 2>/dev/null
}
chart_version() {
local helm_release=
helm_release="${1}"
yq eval .spec.chart.spec.version "${helm_release}" 2>/dev/null
}
chart_values() {
local helm_release=
helm_release="${1}"
yq eval .spec.values "${helm_release}" 2>/dev/null
}