shpaq/auricom-home-cluster
1
0
Fork 0
mirror of https://github.com/auricom/home-cluster.git synced 2025-09-17 18:24:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

🔥 vector archive

Browse Source
This commit is contained in:
auricom
2024-01-25 00:39:58 +01:00
parent 72425b2b55
commit c39864f6fa
8 changed files with 0 additions and 435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
.archive/kubernetes/vector/agent/helmrelease.yaml
View File

@@ -1,84 +0,0 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: &app vector-agent
namespace: monitoring
spec:
interval: 30m
chart:
spec:
chart: vector
version: 0.26.0
sourceRef:
kind: HelmRepository
name: vector
namespace: flux-system
maxHistory: 2
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
image:
repository: timberio/vector
tag: 0.33.0-debian
role: "Agent"
podAnnotations:
configmap.reloader.stakater.com/reload: vector-agent
customConfig:
data_dir: /vector-data-dir
api:
enabled: false
# Sources
sources:
kubernetes_logs:
type: kubernetes_logs
talos_kernel_logs:
type: socket
mode: udp
address: 127.0.0.1:12000
talos_service_logs:
type: socket
mode: udp
address: 127.0.0.1:12001
# Sinks
sinks:
kubernetes_sink:
type: vector
inputs:
- kubernetes_logs
address: "vector-aggregator.monitoring:6000"
version: "2"
talos_kernel_sink:
type: vector
inputs:
- talos_kernel_logs
address: "vector-aggregator.monitoring:6050"
version: "2"
talos_service_sink:
type: vector
inputs:
- talos_service_logs
address: "vector-aggregator.monitoring:6051"
version: "2"
podMonitor:
enabled: true
resources:
requests:
cpu: 23m
memory: 249M
limits:
memory: 918M
service:
enabled: false
tolerations:
- key: node-role.kubernetes.io/control-plane
effect: NoSchedule

7
.archive/kubernetes/vector/agent/kustomization.yaml
View File

@@ -1,7 +0,0 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- ./helmrelease.yaml

160
.archive/kubernetes/vector/aggregator/config/vector.yaml
View File

@@ -1,160 +0,0 @@
data_dir: /vector-data-dir
api:
enabled: true
address: 0.0.0.0:8686
enrichment_tables:
geoip_table:
type: geoip
path: /usr/share/GeoIP/GeoLite2-City.mmdb
# Sources
sources:
kubernetes_source:
address: 0.0.0.0:6000
type: vector
version: "2"
opnsense_logs:
address: 0.0.0.0:6001
type: vector
version: "2"
journald_source:
type: vector
address: 0.0.0.0:6002
version: "2"
vector_metrics:
type: internal_metrics
talos_kernel_logs:
address: 0.0.0.0:6050
type: socket
mode: udp
max_length: 102400
decoding:
codec: json
host_key: __host
talos_service_logs:
address: 0.0.0.0:6051
type: socket
mode: udp
max_length: 102400
decoding:
codec: json
host_key: __host
# Transformations
transforms:
talos_kernel_logs_xform:
type: remap
inputs:
- talos_kernel_logs
source: |-
.__host = replace!(.__host, "192.168.8.101", "talos-node-1")
.__host = replace(.__host, "192.168.8.102", "talos-node-2")
.__host = replace(.__host, "192.168.8.103", "talos-node-3")
.__host = replace(.__host, "192.168.8.104", "talos-node-4")
talos_service_logs_xform:
type: remap
inputs:
- talos_service_logs
source: |-
.__host = replace!(.__host, "192.168.8.101", "talos-node-1")
.__host = replace(.__host, "192.168.8.102", "talos-node-2")
.__host = replace(.__host, "192.168.8.103", "talos-node-3")
.__host = replace(.__host, "192.168.8.104", "talos-node-4")
kubernetes_remap:
type: remap
inputs:
- kubernetes_source
source: |
# Standardize 'app' index
.custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown"
# Sinks
sinks:
loki_kubernetes:
type: loki
inputs:
- kubernetes_source
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
encoding:
codec: json
batch:
max_bytes: 2049000
out_of_order_action: rewrite_timestamp
remove_label_fields: true
remove_timestamp: true
labels:
k8s_app: '{{ custom_app_name }}'
k8s_container: '{{ kubernetes.container_name }}'
k8s_filename: '{{ kubernetes.file }}'
k8s_instance: '{{ kubernetes.pod_labels."app.kubernetes.io/instance" }}'
k8s_namespace: '{{ kubernetes.pod_namespace }}'
k8s_node: '{{ kubernetes.pod_node_name }}'
k8s_pod: '{{ kubernetes.pod_name }}'
loki_opnsense:
type: loki
inputs:
- opnsense_logs
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
encoding:
codec: json
batch:
max_bytes: 400000
out_of_order_action: rewrite_timestamp
labels:
hostname: '{{ host }}'
syslog_identifier: '{{SYSLOG_IDENTIFIER }}'
loki_journal:
type: loki
inputs:
- journald_source
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
encoding:
codec: json
batch:
max_bytes: 2049000
out_of_order_action: accept
remove_label_fields: true
remove_timestamp: true
labels:
hostname: '{{ host }}'
talos_kernel:
type: loki
inputs:
- talos_kernel_logs_xform
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
encoding:
codec: json
except_fields:
- __host
batch:
max_bytes: 1048576
out_of_order_action: rewrite_timestamp
labels:
hostname: '{{ __host }}'
service: '{{ facility }}'
talos_service:
type: loki
inputs:
- talos_service_logs_xform
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
encoding:
codec: json
except_fields:
- __host
batch:
max_bytes: 524288
out_of_order_action: rewrite_timestamp
labels:
hostname: '{{ __host }}'
service: "talos-service"
namespace: "talos:service"

78
.archive/kubernetes/vector/aggregator/helmrelease.yaml
View File

@@ -1,78 +0,0 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: &app vector-aggregator
namespace: monitoring
spec:
interval: 30m
chart:
spec:
chart: app-template
version: 1.5.1
sourceRef:
kind: HelmRepository
name: bjw-s
namespace: flux-system
maxHistory: 2
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
controller:
strategy: RollingUpdate
annotations:
reloader.stakater.com/auto: "true"
image:
repository: docker.io/timberio/vector
tag: 0.33.0-debian
args: ["--config", "/etc/vector/vector.yaml"]
service:
main:
type: LoadBalancer
loadBalancerIP: "${CLUSTER_LB_VECTOR}"
externalTrafficPolicy: Local
ports:
http:
port: 8686
kubernetes-logs:
enabled: true
port: 6000
opnsense-logs:
enabled: true
port: 6001
journald-logs:
enabled: true
port: 6002
talos-kernel:
enabled: true
port: 6050
protocol: UDP
talos-service:
enabled: true
port: 6051
protocol: UDP
persistence:
config:
enabled: true
type: configMap
name: vector-aggregator-configmap
subPath: vector.yaml
mountPath: /etc/vector/vector.yaml
readOnly: true
data:
enabled: true
type: emptyDir
mountPath: /vector-data-dir
geoip:
enabled: true
type: emptyDir
mountPath: /usr/share/GeoIP

15
.archive/kubernetes/vector/aggregator/kustomization.yaml
View File

@@ -1,15 +0,0 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/kustomization.json
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- ./secret.sops.yaml
- ./helmrelease.yaml
configMapGenerator:
- files:
- vector.yaml=./config/vector.yaml
name: vector-aggregator-configmap
generatorOptions:
disableNameSuffixHash: true
patches:
- path: ./patches/geoip.yaml

25
.archive/kubernetes/vector/aggregator/patches/geoip.yaml
View File

@@ -1,25 +0,0 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: vector-aggregator
namespace: monitoring
spec:
values:
initContainers:
init-geoip:
image: docker.io/maxmindinc/geoipupdate:v6.0
env:
- name: GEOIPUPDATE_EDITION_IDS
value: GeoLite2-City
- name: GEOIPUPDATE_FREQUENCY
value: "0"
- name: GEOIPUPDATE_VERBOSE
value: "true"
envFrom:
- secretRef:
name: vector-aggregator-secret
volumeMounts:
- name: geoip
mountPath: /usr/share/GeoIP

30
.archive/kubernetes/vector/aggregator/secret.sops.yaml
View File

@@ -1,30 +0,0 @@
# yamllint disable
apiVersion: v1
kind: Secret
metadata:
name: vector-aggregator-secret
namespace: monitoring
type: Opaque
stringData:
GEOIPUPDATE_ACCOUNT_ID: ENC[AES256_GCM,data:vBU+Iwuv,iv:cK005QUa8iKK+2M2OsKvCXJAkUyhUgReDw8hBBhcNLQ=,tag:k3vrqqyMkp8cnGWfeLbu0A==,type:str]
GEOIPUPDATE_LICENSE_KEY: ENC[AES256_GCM,data:XuCipRddaBHI2umUb1+SPA==,iv:gwbTaK5KCmTF+8mQNjkmLkTdSqz2uFAINo6rJ6F2R4U=,tag:cvevnXWf7xFcdMkwKRF4pQ==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1hhurqwmfvl9m3vh3hk8urulfzcdsrep2ax2neazqt435yhpamu3qj20asg
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlUHFQcWJaRTlGT2RLK3R3
YlJDVTMvRThTR1dXdGN5a1RQd2FxTy84SFdNCnFEWEVpU1o3Y2hISkJrNzBMZFYr
emZyeW9ySnZEYnlvMWFQeXpYeHMzeUkKLS0tIEtPTm9JM0o0ZVBKN05oa0JSbHBL
b2pLSXUyS2lCbmZYYmk0WnVpRU9xRUUKAMUoEprOuR/xgtHZDBmDNTrLEyD9vbeb
dvQZ/7KrgRKVq4Eq3wI254CvajnNs3mACp175DhTsLyX0hBO77FZ2A==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2022-09-15T03:04:22Z"
mac: ENC[AES256_GCM,data:rDDMbtb8xSULRF6RUSNl+Pw4KIiCXJZ5kQ70U5Ap3oB3Ci6miw0EXAVCZC699iJ2YS8cqhUe6VwRCdVn+1bYxz4Dbjm1/dAvkXNbBruhe6KhwSpF/sx6viVH2238ReG+jHr7l/AXVDYyWCxH7hzHWn2f2hTqncpuvr1uyyhU0kg=,iv:JN6F4XDLypDyw9UX9WnhJu+UZzR/A9IW+8NtP4QXnWU=,tag:s+F3V/DNNlvTjFWgjxefoA==,type:str]
pgp: []
encrypted_regex: ^(data|stringData)$
version: 3.7.3

36
.archive/kubernetes/vector/ks.yaml
View File

@@ -1,36 +0,0 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cluster-apps-vector-aggregator
namespace: flux-system
spec:
dependsOn:
- name: cluster-apps-loki-app
path: ./kubernetes/apps/monitoring/vector/aggregator
prune: true
sourceRef:
kind: GitRepository
name: home-ops-kubernetes
interval: 30m
retryInterval: 1m
timeout: 3m
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cluster-apps-vector-agent
namespace: flux-system
spec:
dependsOn:
- name: cluster-apps-vector-aggregator
path: ./kubernetes/apps/monitoring/vector/agent
prune: true
sourceRef:
kind: GitRepository
name: home-ops-kubernetes
interval: 30m
retryInterval: 1m
timeout: 3m