mirror of
https://github.com/auricom/home-cluster.git
synced 2025-09-17 18:24:14 +02:00
🚑 vector aggregator
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
data_dir: /vector-data-dir
|
||||
|
||||
api:
|
||||
enabled: true
|
||||
address: 0.0.0.0:8686
|
||||
|
||||
enrichment_tables:
|
||||
geoip_table:
|
||||
type: geoip
|
||||
path: /geoip/GeoLite2-City.mmdb
|
||||
|
||||
# Sources
|
||||
sources:
|
||||
kubernetes_source:
|
||||
address: 0.0.0.0:6000
|
||||
type: vector
|
||||
version: "2"
|
||||
|
||||
opnsense_logs:
|
||||
address: 0.0.0.0:6001
|
||||
type: vector
|
||||
version: "2"
|
||||
|
||||
journald_source:
|
||||
type: vector
|
||||
address: 0.0.0.0:6002
|
||||
version: "2"
|
||||
|
||||
vector_metrics:
|
||||
type: internal_metrics
|
||||
|
||||
talos_kernel_logs:
|
||||
address: 0.0.0.0:6050
|
||||
type: socket
|
||||
mode: udp
|
||||
max_length: 102400
|
||||
decoding:
|
||||
codec: json
|
||||
host_key: __host
|
||||
|
||||
talos_service_logs:
|
||||
address: 0.0.0.0:6051
|
||||
type: socket
|
||||
mode: udp
|
||||
max_length: 102400
|
||||
decoding:
|
||||
codec: json
|
||||
host_key: __host
|
||||
|
||||
# Transformations
|
||||
transforms:
|
||||
talos_kernel_logs_xform:
|
||||
type: remap
|
||||
inputs:
|
||||
- talos_kernel_logs
|
||||
source: |-
|
||||
.__host = replace!(.__host, "10.1.1.31", "delta")
|
||||
.__host = replace(.__host, "10.1.1.32", "enigma")
|
||||
.__host = replace(.__host, "10.1.1.33", "felix")
|
||||
talos_service_logs_xform:
|
||||
type: remap
|
||||
inputs:
|
||||
- talos_service_logs
|
||||
source: |-
|
||||
.__host = replace!(.__host, "10.1.1.31", "delta")
|
||||
.__host = replace(.__host, "10.1.1.32", "enigma")
|
||||
.__host = replace(.__host, "10.1.1.33", "felix")
|
||||
kubernetes_remap:
|
||||
type: remap
|
||||
inputs:
|
||||
- kubernetes_source
|
||||
source: |
|
||||
# Standardize 'app' index
|
||||
.custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown"
|
||||
# Sinks
|
||||
sinks:
|
||||
loki_kubernetes:
|
||||
type: loki
|
||||
inputs:
|
||||
- kubernetes_source
|
||||
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
|
||||
encoding:
|
||||
codec: json
|
||||
batch:
|
||||
max_bytes: 2049000
|
||||
out_of_order_action: rewrite_timestamp
|
||||
remove_label_fields: true
|
||||
remove_timestamp: true
|
||||
labels:
|
||||
k8s_app: '{{ custom_app_name }}'
|
||||
k8s_container: '{{ kubernetes.container_name }}'
|
||||
k8s_filename: '{{ kubernetes.file }}'
|
||||
k8s_instance: '{{ kubernetes.pod_labels."app.kubernetes.io/instance" }}'
|
||||
k8s_namespace: '{{ kubernetes.pod_namespace }}'
|
||||
k8s_node: '{{ kubernetes.pod_node_name }}'
|
||||
k8s_pod: '{{ kubernetes.pod_name }}'
|
||||
|
||||
loki_opnsense:
|
||||
type: loki
|
||||
inputs:
|
||||
- opnsense_logs
|
||||
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
|
||||
encoding:
|
||||
codec: json
|
||||
batch:
|
||||
max_bytes: 400000
|
||||
out_of_order_action: rewrite_timestamp
|
||||
labels:
|
||||
hostname: '{{ host }}'
|
||||
syslog_identifier: '{{SYSLOG_IDENTIFIER }}'
|
||||
|
||||
loki_journal:
|
||||
type: loki
|
||||
inputs:
|
||||
- journald_source
|
||||
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
|
||||
encoding:
|
||||
codec: json
|
||||
batch:
|
||||
max_bytes: 2049000
|
||||
out_of_order_action: accept
|
||||
remove_label_fields: true
|
||||
remove_timestamp: true
|
||||
labels:
|
||||
hostname: '{{ host }}'
|
||||
|
||||
talos_kernel:
|
||||
type: loki
|
||||
inputs:
|
||||
- talos_kernel_logs_xform
|
||||
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
|
||||
encoding:
|
||||
codec: json
|
||||
except_fields:
|
||||
- __host
|
||||
batch:
|
||||
max_bytes: 1048576
|
||||
out_of_order_action: rewrite_timestamp
|
||||
labels:
|
||||
hostname: '{{ __host }}'
|
||||
service: '{{ facility }}'
|
||||
|
||||
talos_service:
|
||||
type: loki
|
||||
inputs:
|
||||
- talos_service_logs_xform
|
||||
endpoint: http://loki-gateway.monitoring.svc.cluster.local:80
|
||||
encoding:
|
||||
codec: json
|
||||
except_fields:
|
||||
- __host
|
||||
batch:
|
||||
max_bytes: 524288
|
||||
out_of_order_action: rewrite_timestamp
|
||||
labels:
|
||||
hostname: '{{ __host }}'
|
||||
service: "talos-service"
|
||||
namespace: "talos:service"
|
@@ -1,20 +0,0 @@
|
||||
#
|
||||
# IPv4: TCP
|
||||
# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>tcp),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<source_port>[^,]*),(?P<destination_port>[^,]*),(?P<data_length>[^,]*),(?P<tcp_flags>[^,]*),(?P<sequence_number>[^,]*),(?P<ack_number>[^,]*),(?P<tcp_window>[^,]*),(?P<urg>[^,]*),(?P<tcp_options>[^,]*))$
|
||||
# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,62,16800,0,DF,6,tcp,60,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,11715,443,0,S,3876953207,,64240,,mss;sackOK;TS;nop;wscale
|
||||
#
|
||||
# IPv6: TCP
|
||||
# Regex: ?
|
||||
# Example: ?
|
||||
#
|
||||
# IPv4 / IPv6: UDP
|
||||
# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>udp),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<source_port>[^,]*),(?P<destination_port>[^,]*),(?P<data_length>[^,]*))$
|
||||
# Example: 90,,,91e2443ae2e8caf012f9a6e5a8a455c8,lo0,match,pass,in,4,0x4,,255,4660,0,none,17,udp,914,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,5353,5353,894
|
||||
# Example: 15,,,91515c100a3692cb94121964974ce513,igb1_vlan150,match,block,in,6,0x00,0x00000,255,udp,17,391,xxxx::xxxx:xxxx:xxxx:xxxx,xxxx::xx,5353,5353,391
|
||||
#
|
||||
# IPv4: ICMP / IGMP / GRE
|
||||
# Regex: ^(?P<message>(?P<rule>[^,]*),(?P<sub_rule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<interface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ip_version>[^,]*),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>[^,]*),(?P<id>[^,]*),(?P<offset>[^,]*),(?P<flags>[^,]*),(?P<protocol_id>[^,]*),(?P<protocol>icmp|igmp|gre),(?P<length>[^,]*),(?P<source_ip>[^,]*),(?P<destination_ip>[^,]*),(?P<data>[^,]*))$
|
||||
# Example: 94,,,ef794793b2e3764b938bd04cba88e8a3,igb0,match,pass,out,4,0x0,,63,44871,0,DF,1,icmp,84,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=64
|
||||
# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb1_vlan150,match,block,in,4,0xc0,,1,15472,0,none,2,igmp,32,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=8
|
||||
# Example: 16,,,02f4bab031b57d1e30553ce08e0ec131,igb0,match,block,in,4,0x0,,57,20354,0,DF,47,gre,564,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,datalength=544
|
||||
#
|
@@ -2,7 +2,7 @@
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2beta1
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: vector-aggregator
|
||||
name: &app vector-aggregator
|
||||
namespace: monitoring
|
||||
spec:
|
||||
interval: 15m
|
||||
|
@@ -3,3 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./helm-release.yaml
|
||||
configMapGenerator:
|
||||
- name: vector-aggregator-configmap
|
||||
namespace: monitoring
|
||||
files:
|
||||
- vector.yaml=./config/vector.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
Reference in New Issue
Block a user