diff --git a/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml b/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml index 68c5e6b78..9365e4203 100644 --- a/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml +++ b/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml @@ -2,17 +2,17 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: vector-agent + name: &app vector-agent namespace: monitoring spec: - interval: 30m + interval: 15m chart: spec: - chart: vector - version: 0.18.0 + chart: app-template + version: 1.2.0 sourceRef: kind: HelmRepository - name: vector + name: bjw-s namespace: flux-system install: createNamespace: true @@ -21,64 +21,42 @@ spec: upgrade: remediation: retries: 5 - dependsOn: - - name: loki - namespace: monitoring - - name: vector-aggregator - namespace: monitoring values: + controller: + strategy: RollingUpdate + annotations: + reloader.stakater.com/auto: "true" image: - repository: timberio/vector + repository: docker.io/timberio/vector tag: 0.26.0-debian - role: "Agent" - podAnnotations: - configmap.reloader.stakater.com/reload: vector-agent - customConfig: - data_dir: /vector-data-dir - api: - enabled: false - # Sources - sources: - kubernetes_logs: - type: kubernetes_logs - talos_kernel_logs: - type: socket - mode: udp - address: 127.0.0.1:12000 - talos_service_logs: - type: socket - mode: udp - address: 127.0.0.1:12001 - # Sinks - sinks: - kubernetes_sink: - type: vector - inputs: - - kubernetes_logs - address: "vector-aggregator.monitoring:6000" - version: "2" - talos_kernel_sink: - type: vector - inputs: - - talos_kernel_logs - address: "vector-aggregator.monitoring:6050" - version: "2" - talos_service_sink: - type: vector - inputs: - - talos_service_logs - address: "vector-aggregator.monitoring:6051" - version: "2" - podMonitor: - enabled: true + args: ["--config", "/etc/vector/vector.yaml"] + service: + main: + type: LoadBalancer + loadBalancerIP: "${CLUSTER_LB_VECTOR}" + externalTrafficPolicy: Local + ports: + http: + port: 8686 + + persistence: + config: + enabled: true + type: configMap + name: vector-aggregator-configmap + subPath: vector.yaml + mountPath: /etc/vector/vector.yaml + readOnly: true + data: + enabled: true + type: emptyDir + mountPath: /vector-data-dir + geoip: + enabled: true + existingClaim: vector-geoipupdate-config resources: requests: - cpu: 23m - memory: 249M + cpu: 35m + memory: 381M limits: - memory: 918M - service: - enabled: false - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule + memory: 726M diff --git a/kubernetes/cluster-0/apps/logs/vector/aggregator/config/vector.yaml b/kubernetes/cluster-0/apps/logs/vector/aggregator/config/vector.yaml new file mode 100644 index 000000000..e49693ef8 --- /dev/null +++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/config/vector.yaml @@ -0,0 +1,158 @@ +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +enrichment_tables: + geoip_table: + type: geoip + path: /geoip/GeoLite2-City.mmdb + +# Sources +sources: + kubernetes_source: + address: 0.0.0.0:6000 + type: vector + version: "2" + + opnsense_logs: + address: 0.0.0.0:6001 + type: vector + version: "2" + + journald_source: + type: vector + address: 0.0.0.0:6002 + version: "2" + + vector_metrics: + type: internal_metrics + + talos_kernel_logs: + address: 0.0.0.0:6050 + type: socket + mode: udp + max_length: 102400 + decoding: + codec: json + host_key: __host + + talos_service_logs: + address: 0.0.0.0:6051 + type: socket + mode: udp + max_length: 102400 + decoding: + codec: json + host_key: __host + +# Transformations +transforms: + talos_kernel_logs_xform: + type: remap + inputs: + - talos_kernel_logs + source: |- + .__host = replace!(.__host, "10.1.1.31", "delta") + .__host = replace(.__host, "10.1.1.32", "enigma") + .__host = replace(.__host, "10.1.1.33", "felix") + talos_service_logs_xform: + type: remap + inputs: + - talos_service_logs + source: |- + .__host = replace!(.__host, "10.1.1.31", "delta") + .__host = replace(.__host, "10.1.1.32", "enigma") + .__host = replace(.__host, "10.1.1.33", "felix") + kubernetes_remap: + type: remap + inputs: + - kubernetes_source + source: | + # Standardize 'app' index + .custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown" +# Sinks +sinks: + loki_kubernetes: + type: loki + inputs: + - kubernetes_source + endpoint: http://loki-gateway.monitoring.svc.cluster.local:80 + encoding: + codec: json + batch: + max_bytes: 2049000 + out_of_order_action: rewrite_timestamp + remove_label_fields: true + remove_timestamp: true + labels: + k8s_app: '{{ custom_app_name }}' + k8s_container: '{{ kubernetes.container_name }}' + k8s_filename: '{{ kubernetes.file }}' + k8s_instance: '{{ kubernetes.pod_labels."app.kubernetes.io/instance" }}' + k8s_namespace: '{{ kubernetes.pod_namespace }}' + k8s_node: '{{ kubernetes.pod_node_name }}' + k8s_pod: '{{ kubernetes.pod_name }}' + + loki_opnsense: + type: loki + inputs: + - opnsense_logs + endpoint: http://loki-gateway.monitoring.svc.cluster.local:80 + encoding: + codec: json + batch: + max_bytes: 400000 + out_of_order_action: rewrite_timestamp + labels: + hostname: '{{ host }}' + syslog_identifier: '{{SYSLOG_IDENTIFIER }}' + + loki_journal: + type: loki + inputs: + - journald_source + endpoint: http://loki-gateway.monitoring.svc.cluster.local:80 + encoding: + codec: json + batch: + max_bytes: 2049000 + out_of_order_action: accept + remove_label_fields: true + remove_timestamp: true + labels: + hostname: '{{ host }}' + + talos_kernel: + type: loki + inputs: + - talos_kernel_logs_xform + endpoint: http://loki-gateway.monitoring.svc.cluster.local:80 + encoding: + codec: json + except_fields: + - __host + batch: + max_bytes: 1048576 + out_of_order_action: rewrite_timestamp + labels: + hostname: '{{ __host }}' + service: '{{ facility }}' + + talos_service: + type: loki + inputs: + - talos_service_logs_xform + endpoint: http://loki-gateway.monitoring.svc.cluster.local:80 + encoding: + codec: json + except_fields: + - __host + batch: + max_bytes: 524288 + out_of_order_action: rewrite_timestamp + labels: + hostname: '{{ __host }}' + service: "talos-service" + namespace: "talos:service" diff --git a/kubernetes/cluster-0/apps/logs/vector/aggregator/filterlog-regex.txt b/kubernetes/cluster-0/apps/logs/vector/aggregator/filterlog-regex.txt deleted file mode 100644 index 59d572d03..000000000 --- a/kubernetes/cluster-0/apps/logs/vector/aggregator/filterlog-regex.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# IPv4: TCP -# Regex: ^(?P(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?Ptcp),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*))$ -# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,62,16800,0,DF,6,tcp,60,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,11715,443,0,S,3876953207,,64240,,mss;sackOK;TS;nop;wscale -# -# IPv6: TCP -# Regex: ? -# Example: ? -# -# IPv4 / IPv6: UDP -# Regex: ^(?P(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?Pudp),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*))$ -# Example: 90,,,91e2443ae2e8caf012f9a6e5a8a455c8,lo0,match,pass,in,4,0x4,,255,4660,0,none,17,udp,914,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,5353,5353,894 -# Example: 15,,,91515c100a3692cb94121964974ce513,igb1_vlan150,match,block,in,6,0x00,0x00000,255,udp,17,391,xxxx::xxxx:xxxx:xxxx:xxxx,xxxx::xx,5353,5353,391 -# -# IPv4: ICMP / IGMP / GRE -# Regex: ^(?P(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?Picmp|igmp|gre),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[^,]*))$ -# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,63,44871,0,DF,1,icmp,84,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=64 -# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb1_vlan150,match,block,in,4,0xc0,,1,15472,0,none,2,igmp,32,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=8 -# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb0,match,block,in,4,0x0,,57,20354,0,DF,47,gre,564,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=544 -# diff --git a/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml b/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml index 3f568c604..3b240a923 100644 --- a/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml +++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml @@ -2,7 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: vector-aggregator + name: &app vector-aggregator namespace: monitoring spec: interval: 15m diff --git a/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml b/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml index bfe61558b..6bcf42430 100644 --- a/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml +++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml @@ -3,3 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./helm-release.yaml +configMapGenerator: + - name: vector-aggregator-configmap + namespace: monitoring + files: + - vector.yaml=./config/vector.yaml +generatorOptions: + disableNameSuffixHash: true