🚑 vector aggregator

2025-09-17 18:24:14 +02:00 · 2022-12-07 18:32:20 +01:00
parent e5fc46fb56
commit ecedb542b8
5 changed files with 204 additions and 81 deletions
--- a/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml
+++ b/kubernetes/cluster-0/apps/logs/vector/agent/helm-release.yaml
@@ -2,17 +2,17 @@
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
-  name: vector-agent
+  name: &app vector-agent
  namespace: monitoring
 spec:
-  interval: 30m
+  interval: 15m
  chart:
    spec:
-      chart: vector
-      version: 0.18.0
+      chart: app-template
+      version: 1.2.0
      sourceRef:
        kind: HelmRepository
-        name: vector
+        name: bjw-s
        namespace: flux-system
  install:
    createNamespace: true
@@ -21,64 +21,42 @@ spec:
  upgrade:
    remediation:
      retries: 5
-  dependsOn:
-    - name: loki
-      namespace: monitoring
-    - name: vector-aggregator
-      namespace: monitoring
  values:
+    controller:
+      strategy: RollingUpdate
+      annotations:
+        reloader.stakater.com/auto: "true"
    image:
-      repository: timberio/vector
+      repository: docker.io/timberio/vector
      tag: 0.26.0-debian
-    role: "Agent"
-    podAnnotations:
-      configmap.reloader.stakater.com/reload: vector-agent
-    customConfig:
-      data_dir: /vector-data-dir
-      api:
-        enabled: false
-      # Sources
-      sources:
-        kubernetes_logs:
-          type: kubernetes_logs
-        talos_kernel_logs:
-          type: socket
-          mode: udp
-          address: 127.0.0.1:12000
-        talos_service_logs:
-          type: socket
-          mode: udp
-          address: 127.0.0.1:12001
-      # Sinks
-      sinks:
-        kubernetes_sink:
-          type: vector
-          inputs:
-            - kubernetes_logs
-          address: "vector-aggregator.monitoring:6000"
-          version: "2"
-        talos_kernel_sink:
-          type: vector
-          inputs:
-            - talos_kernel_logs
-          address: "vector-aggregator.monitoring:6050"
-          version: "2"
-        talos_service_sink:
-          type: vector
-          inputs:
-            - talos_service_logs
-          address: "vector-aggregator.monitoring:6051"
-          version: "2"
-    podMonitor:
+    args: ["--config", "/etc/vector/vector.yaml"]
+    service:
+      main:
+        type: LoadBalancer
+        loadBalancerIP: "${CLUSTER_LB_VECTOR}"
+        externalTrafficPolicy: Local
+        ports:
+          http:
+            port: 8686
+
+    persistence:
+      config:
        enabled: true
+        type: configMap
+        name: vector-aggregator-configmap
+        subPath: vector.yaml
+        mountPath: /etc/vector/vector.yaml
+        readOnly: true
+      data:
+        enabled: true
+        type: emptyDir
+        mountPath: /vector-data-dir
+      geoip:
+        enabled: true
+        existingClaim: vector-geoipupdate-config
    resources:
      requests:
-        cpu: 23m
-        memory: 249M
+        cpu: 35m
+        memory: 381M
      limits:
-        memory: 918M
-    service:
-      enabled: false
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        effect: NoSchedule
+        memory: 726M
--- a/kubernetes/cluster-0/apps/logs/vector/aggregator/config/vector.yaml
+++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/config/vector.yaml
@@ -0,0 +1,158 @@
+data_dir: /vector-data-dir
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+
+enrichment_tables:
+  geoip_table:
+    type: geoip
+    path: /geoip/GeoLite2-City.mmdb
+
+# Sources
+sources:
+  kubernetes_source:
+    address: 0.0.0.0:6000
+    type: vector
+    version: "2"
+
+  opnsense_logs:
+    address: 0.0.0.0:6001
+    type: vector
+    version: "2"
+
+  journald_source:
+    type: vector
+    address: 0.0.0.0:6002
+    version: "2"
+
+  vector_metrics:
+    type: internal_metrics
+
+  talos_kernel_logs:
+    address: 0.0.0.0:6050
+    type: socket
+    mode: udp
+    max_length: 102400
+    decoding:
+      codec: json
+    host_key: __host
+
+  talos_service_logs:
+    address: 0.0.0.0:6051
+    type: socket
+    mode: udp
+    max_length: 102400
+    decoding:
+      codec: json
+    host_key: __host
+
+# Transformations
+transforms:
+  talos_kernel_logs_xform:
+    type: remap
+    inputs:
+      - talos_kernel_logs
+    source: |-
+      .__host = replace!(.__host, "10.1.1.31", "delta")
+      .__host = replace(.__host, "10.1.1.32", "enigma")
+      .__host = replace(.__host, "10.1.1.33", "felix")
+  talos_service_logs_xform:
+    type: remap
+    inputs:
+      - talos_service_logs
+    source: |-
+      .__host = replace!(.__host, "10.1.1.31", "delta")
+      .__host = replace(.__host, "10.1.1.32", "enigma")
+      .__host = replace(.__host, "10.1.1.33", "felix")
+  kubernetes_remap:
+    type: remap
+    inputs:
+      - kubernetes_source
+    source: |
+      # Standardize 'app' index
+      .custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown"
+# Sinks
+sinks:
+  loki_kubernetes:
+    type: loki
+    inputs:
+      - kubernetes_source
+    endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
+    encoding:
+      codec: json
+    batch:
+      max_bytes: 2049000
+    out_of_order_action: rewrite_timestamp
+    remove_label_fields: true
+    remove_timestamp: true
+    labels:
+      k8s_app: '{{ custom_app_name }}'
+      k8s_container: '{{ kubernetes.container_name }}'
+      k8s_filename: '{{ kubernetes.file }}'
+      k8s_instance: '{{ kubernetes.pod_labels."app.kubernetes.io/instance" }}'
+      k8s_namespace: '{{ kubernetes.pod_namespace }}'
+      k8s_node: '{{ kubernetes.pod_node_name }}'
+      k8s_pod: '{{ kubernetes.pod_name }}'
+
+  loki_opnsense:
+    type: loki
+    inputs:
+      - opnsense_logs
+    endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
+    encoding:
+      codec: json
+    batch:
+      max_bytes: 400000
+    out_of_order_action: rewrite_timestamp
+    labels:
+      hostname: '{{ host }}'
+      syslog_identifier: '{{SYSLOG_IDENTIFIER }}'
+
+  loki_journal:
+    type: loki
+    inputs:
+      - journald_source
+    endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
+    encoding:
+      codec: json
+    batch:
+      max_bytes: 2049000
+    out_of_order_action: accept
+    remove_label_fields: true
+    remove_timestamp: true
+    labels:
+      hostname: '{{ host }}'
+
+  talos_kernel:
+    type: loki
+    inputs:
+      - talos_kernel_logs_xform
+    endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
+    encoding:
+      codec: json
+      except_fields:
+        - __host
+    batch:
+      max_bytes: 1048576
+    out_of_order_action: rewrite_timestamp
+    labels:
+      hostname: '{{ __host }}'
+      service: '{{ facility }}'
+
+  talos_service:
+    type: loki
+    inputs:
+      - talos_service_logs_xform
+    endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
+    encoding:
+      codec: json
+      except_fields:
+        - __host
+    batch:
+      max_bytes: 524288
+    out_of_order_action: rewrite_timestamp
+    labels:
+      hostname: '{{ __host }}'
+      service: "talos-service"
+      namespace: "talos:service"
--- a/kubernetes/cluster-0/apps/logs/vector/aggregator/filterlog-regex.txt
+++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/filterlog-regex.txt
@@ -1,20 +0,0 @@
-#
-# IPv4: TCP
-# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>tcp),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<source_port>[^,]*),(?P<destination_port>[^,]*),(?P<data_length>[^,]*),(?P<tcp_flags>[^,]*),(?P<sequence_number>[^,]*),(?P<ack_number>[^,]*),(?P<tcp_window>[^,]*),(?P<urg>[^,]*),(?P<tcp_options>[^,]*))$
-# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,62,16800,0,DF,6,tcp,60,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,11715,443,0,S,3876953207,,64240,,mss;sackOK;TS;nop;wscale
-#
-# IPv6: TCP
-# Regex: ?
-# Example: ?
-#
-# IPv4 / IPv6: UDP
-# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>udp),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<source_port>[^,]*),(?P<destination_port>[^,]*),(?P<data_length>[^,]*))$
-# Example: 90,,,91e2443ae2e8caf012f9a6e5a8a455c8,lo0,match,pass,in,4,0x4,,255,4660,0,none,17,udp,914,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,5353,5353,894
-# Example: 15,,,91515c100a3692cb94121964974ce513,igb1_vlan150,match,block,in,6,0x00,0x00000,255,udp,17,391,xxxx::xxxx:xxxx:xxxx:xxxx,xxxx::xx,5353,5353,391
-#
-# IPv4: ICMP / IGMP / GRE
-# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>icmp|igmp|gre),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<data>[^,]*))$
-# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,63,44871,0,DF,1,icmp,84,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=64
-# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb1_vlan150,match,block,in,4,0xc0,,1,15472,0,none,2,igmp,32,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=8
-# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb0,match,block,in,4,0x0,,57,20354,0,DF,47,gre,564,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=544
-#
--- a/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml
+++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/helm-release.yaml
@@ -2,7 +2,7 @@
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
-  name: vector-aggregator
+  name: &app vector-aggregator
  namespace: monitoring
 spec:
  interval: 15m
--- a/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml
+++ b/kubernetes/cluster-0/apps/logs/vector/aggregator/kustomization.yaml
@@ -3,3 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./helm-release.yaml
+configMapGenerator:
+  - name: vector-aggregator-configmap
+    namespace: monitoring
+    files:
+      - vector.yaml=./config/vector.yaml
+generatorOptions:
+  disableNameSuffixHash: true