add loki && syslog-ng

fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng add loki && syslog-ng
2025-10-02 00:34:25 +02:00 · 2020-12-05 17:21:20 +01:00
parent 92f83b38f0
commit cabc5e1760
4 changed files with 291 additions and 0 deletions
--- a/cluster/flux-system/helm-chart-repositories.yaml
+++ b/cluster/flux-system/helm-chart-repositories.yaml
@@ -128,3 +128,13 @@ spec:
  interval: 10m
  url: https://charts.longhorn.io
  timeout: 3m
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: grafana-loki-charts
+  namespace: flux-system
+spec:
+  interval: 10m
+  url: https://grafana.github.io/loki/charts
+  timeout: 3m
--- a/cluster/monitoring/loki-stack.yaml
+++ b/cluster/monitoring/loki-stack.yaml
@@ -0,0 +1,148 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: loki-stack
+  namespace: monitoring
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: loki-stack
+      version: 2.1.1
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-loki-charts
+        namespace: flux-system
+      interval: 5m
+  values:
+    loki:
+      replicas: 3
+      persistence:
+        enabled: false
+      config:
+        auth_enabled: false
+        server:
+          http_listen_port: 3100
+        distributor:
+          ring:
+            kvstore:
+              store: memberlist
+        ingester:
+          lifecycler:
+            ring:
+              kvstore:
+                store: memberlist
+              replication_factor: 1
+            final_sleep: 0s
+          chunk_idle_period: 5m
+          chunk_retain_period: 30s
+        memberlist:
+          abort_if_cluster_join_fails: false
+          # Expose this port on all distributor, ingester
+          # and querier replicas.
+          bind_port: 7946
+          # You can use a headless k8s service for all distributor,
+          # ingester and querier components.
+          join_members:
+          - loki-stack-headless:7946
+          # max_join_backoff: 1m
+          # max_join_retries: 10
+          # min_join_backoff: 1s
+        schema_config:
+          configs:
+          - from: 2020-05-15
+            store: boltdb-shipper
+            object_store: s3
+            schema: v11
+            index:
+              prefix: index_
+              period: 24h
+        storage_config:
+          boltdb_shipper:
+            active_index_directory: /data/loki/index
+            cache_location: /data/loki/index_cache
+            resync_interval: 5s
+            shared_store: s3
+        limits_config:
+          enforce_metric_name: false
+          reject_old_samples: true
+          reject_old_samples_max_age: 168h
+      extraPorts:
+      - port: 7956
+        protocol: TCP
+        name: loki-gossip-ring
+        targetPort: 7946
+      serviceMonitor:
+        enabled: true
+      podAnnotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "http-metrics"
+    promtail:
+      serviceMonitor:
+        enabled: true
+      extraScrapeConfigs:
+      pipeline_stages:
+        - job_name: pfsense
+          syslog:
+            listen_address: 0.0.0.0:1514
+            idle_timeout: 60s
+            label_structured_data: false
+            labels:
+              job: "syslog"
+              host: pfsense
+          relabel_configs:
+            - source_labels: ["__syslog_message_severity"]
+              target_label: "severity"
+            #- source_labels: ['__syslog_message_facility']
+            #  target_label: 'facility'
+            - source_labels: ["__syslog_message_app_name"]
+              target_label: "app_name"
+          pipeline_stages:
+            - match:
+                selector: '{app_name="filterlog"}'
+                stages:
+                  - regex:
+                      expression: '(?P<pfsense_fw_rule>\d*?),(?P<pfsense_fw_subrule>\d*?),(?P<pfsense_fw_anchor>\d*?),(?P<pfsense_fw_tracker>\d*?),(?P<pfsense_fw_interface>igb.{1,5}?),(?P<pfsense_fw_reason>\w*?),(?P<pfsense_fw_action>\w*?),(?P<pfsense_fw_direction>\w*?),(?P<pfsense_fw_ip_version>4{1}?),(?P<pfsense_fw_tos>\w*?),(?P<pfsense_fw_ecn>\w*?),(?P<pfsense_fw_ttl>\w*?),(?P<pfsense_fw_id>\w*?),(?P<pfsense_fw_offset>\w*?),(?P<pfsense_fw_flag>\w*?),(?P<pfsense_fw_protocol_id>\d*?),(?P<pfsense_fw_protocol_text>\w*?),(?P<pfsense_fw_length>\d*?),(?P<pfsense_fw_source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}?),(?P<pfsense_fw_destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}?),(?P<pfsense_fw_source_port>\d+?),(?P<pfsense_fw_destination_port>\d+?),(?P<pfsense_fw_data_length>\d+?)'
+                      # ipv6 // ,(?P<pfsense_fw_ip_version>6{1}?),(?P<pfsense_fw_lass>\w*?),(?P<pfsense_fw_flow_label>\w*?),(?P<pfsense_fw_hop_limit>\w*?),(?P<pfsense_fw_protocol_text>\w*?),(?P<pfsense_fw_protocol_id>\d*?),
+                  - labels:
+                      pfsense_fw_rule: ""
+                      #pfsense_fw_subrule: ''
+                      #pfsense_fw_anchor: ''
+                      pfsense_fw_tracker: ""
+                      pfsense_fw_interface: ""
+                      pfsense_fw_reason: ""
+                      pfsense_fw_action: ""
+                      pfsense_fw_direction: ""
+                      #pfsense_fw_ip_version: ''
+                      #pfsense_fw_tos: ''
+                      #pfsense_fw_ecn: ''
+                      #pfsense_fw_ttl: ''
+                      #pfsense_fw_id: ''
+                      #pfsense_fw_offset: ''
+                      #pfsense_fw_flag: ''
+                      pfsense_fw_protocol_id: ""
+                      pfsense_fw_protocol_text: ""
+                      #pfsense_fw_length: ''
+                      pfsense_fw_source_address: ""
+                      pfsense_fw_destination_address: ""
+                      pfsense_fw_source_port: ""
+                      pfsense_fw_destination_port: ""
+                      #pfsense_fw_data_length: ''
+      #              - metrics:
+      #                  lines_total:
+      #                    type: Counter
+      #                    description: "pfsense firewall : total number of log lines"
+      #                    prefix: pfsense_firewall_
+      #                    match_all: true
+      #                    count_entry_bytes: true
+      #                    config:
+      #                      action: add
+      syslogService:
+        enabled: true
+        type: LoadBalancer
+        port: 1514
+        loadBalancerIP: 192.168.9.208
+  valuesFrom:
+  - kind: ConfigMap
+    name: helmrelease-monitoring-loki-stack
--- a/cluster/monitoring/syslog-ng.yaml
+++ b/cluster/monitoring/syslog-ng.yaml
@@ -0,0 +1,97 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: syslog-ng
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/instance: syslog-ng
+    app.kubernetes.io/name: syslog-ng
+  annotations:
+    fluxcd.io/ignored: "false"
+    fluxcd.io/automated: "true"
+    fluxcd.io/tag.syslog-ng: semver:*
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: syslog-ng
+      app.kubernetes.io/name: syslog-ng
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: syslog-ng
+        app.kubernetes.io/name: syslog-ng
+    spec:
+      containers:
+      - image: balabit/syslog-ng:3.29.1
+        imagePullPolicy: Always
+        name: syslog-ng
+          #          securityContext:
+          #            capabilities:
+          #              drop:
+          #                - ALL
+          #            readOnlyRootFilesystem: true
+          #            runAsNonRoot: true
+          #            runAsUser: 1001
+        ports:
+        - containerPort: 514
+          name: pfsense-syslog
+        volumeMounts:
+        - name: config
+          mountPath: /etc/syslog-ng/syslog-ng.conf
+          subPath: syslog-ng.conf
+        livenessProbe:
+          exec:
+            command:
+            - cat
+      volumes:
+      - name: config
+        configMap:
+          name: syslog-ng-config
+      dnsConfig:
+        options:
+        - name: ndots
+          value: "1"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: syslog-ng-config
+  namespace: monitoring
+data:
+  syslog-ng.conf: |
+    @version: 3.27
+
+    source pfsense {
+      udp(ip(0.0.0.0) port(514) flags(no-hostname));
+    };
+
+    destination loki_syslog {
+      syslog("loki-promtail-syslog" transport("tcp") port(1514));
+    };
+
+    log {
+      source(pfsense);
+      destination(loki_syslog);
+    };
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/instance: syslog-ng
+    app.kubernetes.io/name: syslog-ng
+  name: syslog-ng
+  namespace: monitoring
+spec:
+  ports:
+  - name: pfsense-syslog
+    port: 514
+    protocol: UDP
+    targetPort: 514
+  selector:
+    app.kubernetes.io/instance: syslog-ng
+    app.kubernetes.io/name: syslog-ng
+  type: LoadBalancer
+  loadBalancerIP: 192.168.9.202
--- a/secrets/helmrelease-monitoring-loki-stack.yaml
+++ b/secrets/helmrelease-monitoring-loki-stack.yaml
@@ -0,0 +1,36 @@
+apiVersion: v1
+data:
+    values.yaml: ENC[AES256_GCM,data:Tc27M0zdni0zww6C3smnobyGdHD8Vxry+Iz1r1lZWRPK78ZBD47CFwpNiXrIOGDkQYsFrRvRIiVrjDFvsoardKVCRK+k2mW4KY4H8ZApZCd6Wy/7Vy0w65WKl7ZO20U7/Z8AVDl6QGhi4hLRLBqLgupSu9KvupTd34tKeraRzMS/UH4AI77mHvRGKc51KuUCFy99Q0B++rEN1do=,iv:y9tGCCAQ+5JBGn94feC27WcLZefWn9ub1z6HqEBOkQs=,tag:YeidoQ6qDGqubK1o4kscOw==,type:str]
+kind: ConfigMap
+metadata:
+    creationTimestamp: null
+    name: helmrelease-monitoring-loki-stack
+    namespace: monitoring
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    lastmodified: '2020-12-05T17:48:50Z'
+    mac: ENC[AES256_GCM,data:gjui8RaeYoAr78gwe2OIk7cVtxw3PwZDh5uZyi3cG3SDoTOOK+YInj0/xwP3+FpWLs9N7i/QIxYuzs+Vd6+6bOzkY6YjIDJBpdtFSXefCknbkFG8zBVgaRboD7m7txliirQ3qUwQC8wvc9nFuLeBuzo5BSK1v+uNMHwveeyFU7M=,iv:5k0BsC7dBdjPpeA1UnwpDouC6EA3V0Z+P3VhTczis/k=,tag:U8WNduwUNjfEO7jLAkSo7g==,type:str]
+    pgp:
+    -   created_at: '2020-12-05T17:48:50Z'
+        enc: |
+            -----BEGIN PGP MESSAGE-----
+
+            hQGMA/JorPHm1g9XAQv+L90aZXZRKICnvXOQxxhgSEJq/tudWNXt+HfC2/x4L/dO
+            pz6OXz/J20+x3mSOB8J9IkYXcy7NMr3LGYCX/BoupvngI9yGu0i2fj69AYLaQ4zi
+            a8wWH68YJX+FHeWIs8ltG2naYt2MLr/rbADaTpreILimywagXw6ViOwdVDRgh1JZ
+            qQtdDwEWUjD+t9Lve0s63Z6vFx8s6CnA+xqyB5ci84VMCqSmE4EvtN54JcFOzOHE
+            4CJJ2efYpSFZyvRTxwp33MyuZFGs8GF+HQfqy2EhLfmo+g2A5x0egahQptf3cGmi
+            3GcEIul9Oh/uPAd+LPB6zwlVbg849I8o8CbdDyjfHBt36pC54jmUg9YRPHTd+t0z
+            r+g17JXcCb5oBSUs6b5opKdtfH4nscYWFIffZ7LW7A5Tn8HLRcWYMW06+/N1Ft5j
+            TVqUutVreMXnuctXeGKS9/obpUqVpP8Lui5bYoJgCio7q4womxsR1spsNwtPa07u
+            G7knAzmYY73EI2AXLayf0l4BDkby4QPPpZKVR+kYeu7EFrMcMPqGl+IwLN03zHMM
+            xuCbSZ59zzsm/t6FnsfsVHXHILrVFcEH+2pkSAkRJHvBBmWK10KIdUh3Hgd238nf
+            GFv2i8CJiJCx8E8yAikD
+            =0gs7
+            -----END PGP MESSAGE-----
+        fp: C8F8A49D04A1AB639F8EA21CDBA4B1DCB1FA5BDD
+    encrypted_regex: ^(data|stringData)$
+    version: 3.6.1