mirror of
https://github.com/auricom/home-cluster.git
synced 2025-10-02 16:51:52 +02:00
add loki && syslog-ng
fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng fix loki && syslog-ng add loki && syslog-ng
This commit is contained in:
auricom
@@ -128,3 +128,13 @@ spec:
|
|||||||
interval: 10m
|
interval: 10m
|
||||||
url: https://charts.longhorn.io
|
url: https://charts.longhorn.io
|
||||||
timeout: 3m
|
timeout: 3m
|
||||||
|
---
|
||||||
|
apiVersion: source.toolkit.fluxcd.io/v1beta1
|
||||||
|
kind: HelmRepository
|
||||||
|
metadata:
|
||||||
|
name: grafana-loki-charts
|
||||||
|
namespace: flux-system
|
||||||
|
spec:
|
||||||
|
interval: 10m
|
||||||
|
url: https://grafana.github.io/loki/charts
|
||||||
|
timeout: 3m
|
||||||
148
cluster/monitoring/loki-stack.yaml
Normal file
148
cluster/monitoring/loki-stack.yaml
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
---
|
||||||
|
apiVersion: helm.toolkit.fluxcd.io/v2beta1
|
||||||
|
kind: HelmRelease
|
||||||
|
metadata:
|
||||||
|
name: loki-stack
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
interval: 5m
|
||||||
|
chart:
|
||||||
|
spec:
|
||||||
|
chart: loki-stack
|
||||||
|
version: 2.1.1
|
||||||
|
sourceRef:
|
||||||
|
kind: HelmRepository
|
||||||
|
name: grafana-loki-charts
|
||||||
|
namespace: flux-system
|
||||||
|
interval: 5m
|
||||||
|
values:
|
||||||
|
loki:
|
||||||
|
replicas: 3
|
||||||
|
persistence:
|
||||||
|
enabled: false
|
||||||
|
config:
|
||||||
|
auth_enabled: false
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
distributor:
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: memberlist
|
||||||
|
ingester:
|
||||||
|
lifecycler:
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: memberlist
|
||||||
|
replication_factor: 1
|
||||||
|
final_sleep: 0s
|
||||||
|
chunk_idle_period: 5m
|
||||||
|
chunk_retain_period: 30s
|
||||||
|
memberlist:
|
||||||
|
abort_if_cluster_join_fails: false
|
||||||
|
# Expose this port on all distributor, ingester
|
||||||
|
# and querier replicas.
|
||||||
|
bind_port: 7946
|
||||||
|
# You can use a headless k8s service for all distributor,
|
||||||
|
# ingester and querier components.
|
||||||
|
join_members:
|
||||||
|
- loki-stack-headless:7946
|
||||||
|
# max_join_backoff: 1m
|
||||||
|
# max_join_retries: 10
|
||||||
|
# min_join_backoff: 1s
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2020-05-15
|
||||||
|
store: boltdb-shipper
|
||||||
|
object_store: s3
|
||||||
|
schema: v11
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
storage_config:
|
||||||
|
boltdb_shipper:
|
||||||
|
active_index_directory: /data/loki/index
|
||||||
|
cache_location: /data/loki/index_cache
|
||||||
|
resync_interval: 5s
|
||||||
|
shared_store: s3
|
||||||
|
limits_config:
|
||||||
|
enforce_metric_name: false
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
extraPorts:
|
||||||
|
- port: 7956
|
||||||
|
protocol: TCP
|
||||||
|
name: loki-gossip-ring
|
||||||
|
targetPort: 7946
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: true
|
||||||
|
podAnnotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "http-metrics"
|
||||||
|
promtail:
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: true
|
||||||
|
extraScrapeConfigs:
|
||||||
|
pipeline_stages:
|
||||||
|
- job_name: pfsense
|
||||||
|
syslog:
|
||||||
|
listen_address: 0.0.0.0:1514
|
||||||
|
idle_timeout: 60s
|
||||||
|
label_structured_data: false
|
||||||
|
labels:
|
||||||
|
job: "syslog"
|
||||||
|
host: pfsense
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: ["__syslog_message_severity"]
|
||||||
|
target_label: "severity"
|
||||||
|
#- source_labels: ['__syslog_message_facility']
|
||||||
|
# target_label: 'facility'
|
||||||
|
- source_labels: ["__syslog_message_app_name"]
|
||||||
|
target_label: "app_name"
|
||||||
|
pipeline_stages:
|
||||||
|
- match:
|
||||||
|
selector: '{app_name="filterlog"}'
|
||||||
|
stages:
|
||||||
|
- regex:
|
||||||
|
expression: '(?P<pfsense_fw_rule>\d*?),(?P<pfsense_fw_subrule>\d*?),(?P<pfsense_fw_anchor>\d*?),(?P<pfsense_fw_tracker>\d*?),(?P<pfsense_fw_interface>igb.{1,5}?),(?P<pfsense_fw_reason>\w*?),(?P<pfsense_fw_action>\w*?),(?P<pfsense_fw_direction>\w*?),(?P<pfsense_fw_ip_version>4{1}?),(?P<pfsense_fw_tos>\w*?),(?P<pfsense_fw_ecn>\w*?),(?P<pfsense_fw_ttl>\w*?),(?P<pfsense_fw_id>\w*?),(?P<pfsense_fw_offset>\w*?),(?P<pfsense_fw_flag>\w*?),(?P<pfsense_fw_protocol_id>\d*?),(?P<pfsense_fw_protocol_text>\w*?),(?P<pfsense_fw_length>\d*?),(?P<pfsense_fw_source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}?),(?P<pfsense_fw_destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}?),(?P<pfsense_fw_source_port>\d+?),(?P<pfsense_fw_destination_port>\d+?),(?P<pfsense_fw_data_length>\d+?)'
|
||||||
|
# ipv6 // ,(?P<pfsense_fw_ip_version>6{1}?),(?P<pfsense_fw_lass>\w*?),(?P<pfsense_fw_flow_label>\w*?),(?P<pfsense_fw_hop_limit>\w*?),(?P<pfsense_fw_protocol_text>\w*?),(?P<pfsense_fw_protocol_id>\d*?),
|
||||||
|
- labels:
|
||||||
|
pfsense_fw_rule: ""
|
||||||
|
#pfsense_fw_subrule: ''
|
||||||
|
#pfsense_fw_anchor: ''
|
||||||
|
pfsense_fw_tracker: ""
|
||||||
|
pfsense_fw_interface: ""
|
||||||
|
pfsense_fw_reason: ""
|
||||||
|
pfsense_fw_action: ""
|
||||||
|
pfsense_fw_direction: ""
|
||||||
|
#pfsense_fw_ip_version: ''
|
||||||
|
#pfsense_fw_tos: ''
|
||||||
|
#pfsense_fw_ecn: ''
|
||||||
|
#pfsense_fw_ttl: ''
|
||||||
|
#pfsense_fw_id: ''
|
||||||
|
#pfsense_fw_offset: ''
|
||||||
|
#pfsense_fw_flag: ''
|
||||||
|
pfsense_fw_protocol_id: ""
|
||||||
|
pfsense_fw_protocol_text: ""
|
||||||
|
#pfsense_fw_length: ''
|
||||||
|
pfsense_fw_source_address: ""
|
||||||
|
pfsense_fw_destination_address: ""
|
||||||
|
pfsense_fw_source_port: ""
|
||||||
|
pfsense_fw_destination_port: ""
|
||||||
|
#pfsense_fw_data_length: ''
|
||||||
|
# - metrics:
|
||||||
|
# lines_total:
|
||||||
|
# type: Counter
|
||||||
|
# description: "pfsense firewall : total number of log lines"
|
||||||
|
# prefix: pfsense_firewall_
|
||||||
|
# match_all: true
|
||||||
|
# count_entry_bytes: true
|
||||||
|
# config:
|
||||||
|
# action: add
|
||||||
|
syslogService:
|
||||||
|
enabled: true
|
||||||
|
type: LoadBalancer
|
||||||
|
port: 1514
|
||||||
|
loadBalancerIP: 192.168.9.208
|
||||||
|
valuesFrom:
|
||||||
|
- kind: ConfigMap
|
||||||
|
name: helmrelease-monitoring-loki-stack
|
||||||
97
cluster/monitoring/syslog-ng.yaml
Normal file
97
cluster/monitoring/syslog-ng.yaml
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: syslog-ng
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/instance: syslog-ng
|
||||||
|
app.kubernetes.io/name: syslog-ng
|
||||||
|
annotations:
|
||||||
|
fluxcd.io/ignored: "false"
|
||||||
|
fluxcd.io/automated: "true"
|
||||||
|
fluxcd.io/tag.syslog-ng: semver:*
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/instance: syslog-ng
|
||||||
|
app.kubernetes.io/name: syslog-ng
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/instance: syslog-ng
|
||||||
|
app.kubernetes.io/name: syslog-ng
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- image: balabit/syslog-ng:3.29.1
|
||||||
|
imagePullPolicy: Always
|
||||||
|
name: syslog-ng
|
||||||
|
# securityContext:
|
||||||
|
# capabilities:
|
||||||
|
# drop:
|
||||||
|
# - ALL
|
||||||
|
# readOnlyRootFilesystem: true
|
||||||
|
# runAsNonRoot: true
|
||||||
|
# runAsUser: 1001
|
||||||
|
ports:
|
||||||
|
- containerPort: 514
|
||||||
|
name: pfsense-syslog
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/syslog-ng/syslog-ng.conf
|
||||||
|
subPath: syslog-ng.conf
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- cat
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: syslog-ng-config
|
||||||
|
dnsConfig:
|
||||||
|
options:
|
||||||
|
- name: ndots
|
||||||
|
value: "1"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: syslog-ng-config
|
||||||
|
namespace: monitoring
|
||||||
|
data:
|
||||||
|
syslog-ng.conf: |
|
||||||
|
@version: 3.27
|
||||||
|
|
||||||
|
source pfsense {
|
||||||
|
udp(ip(0.0.0.0) port(514) flags(no-hostname));
|
||||||
|
};
|
||||||
|
|
||||||
|
destination loki_syslog {
|
||||||
|
syslog("loki-promtail-syslog" transport("tcp") port(1514));
|
||||||
|
};
|
||||||
|
|
||||||
|
log {
|
||||||
|
source(pfsense);
|
||||||
|
destination(loki_syslog);
|
||||||
|
};
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/instance: syslog-ng
|
||||||
|
app.kubernetes.io/name: syslog-ng
|
||||||
|
name: syslog-ng
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- name: pfsense-syslog
|
||||||
|
port: 514
|
||||||
|
protocol: UDP
|
||||||
|
targetPort: 514
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/instance: syslog-ng
|
||||||
|
app.kubernetes.io/name: syslog-ng
|
||||||
|
type: LoadBalancer
|
||||||
|
loadBalancerIP: 192.168.9.202
|
||||||
36
secrets/helmrelease-monitoring-loki-stack.yaml
Normal file
36
secrets/helmrelease-monitoring-loki-stack.yaml
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
data:
|
||||||
|
values.yaml: ENC[AES256_GCM,data:Tc27M0zdni0zww6C3smnobyGdHD8Vxry+Iz1r1lZWRPK78ZBD47CFwpNiXrIOGDkQYsFrRvRIiVrjDFvsoardKVCRK+k2mW4KY4H8ZApZCd6Wy/7Vy0w65WKl7ZO20U7/Z8AVDl6QGhi4hLRLBqLgupSu9KvupTd34tKeraRzMS/UH4AI77mHvRGKc51KuUCFy99Q0B++rEN1do=,iv:y9tGCCAQ+5JBGn94feC27WcLZefWn9ub1z6HqEBOkQs=,tag:YeidoQ6qDGqubK1o4kscOw==,type:str]
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
creationTimestamp: null
|
||||||
|
name: helmrelease-monitoring-loki-stack
|
||||||
|
namespace: monitoring
|
||||||
|
sops:
|
||||||
|
kms: []
|
||||||
|
gcp_kms: []
|
||||||
|
azure_kv: []
|
||||||
|
hc_vault: []
|
||||||
|
lastmodified: '2020-12-05T17:48:50Z'
|
||||||
|
mac: ENC[AES256_GCM,data:gjui8RaeYoAr78gwe2OIk7cVtxw3PwZDh5uZyi3cG3SDoTOOK+YInj0/xwP3+FpWLs9N7i/QIxYuzs+Vd6+6bOzkY6YjIDJBpdtFSXefCknbkFG8zBVgaRboD7m7txliirQ3qUwQC8wvc9nFuLeBuzo5BSK1v+uNMHwveeyFU7M=,iv:5k0BsC7dBdjPpeA1UnwpDouC6EA3V0Z+P3VhTczis/k=,tag:U8WNduwUNjfEO7jLAkSo7g==,type:str]
|
||||||
|
pgp:
|
||||||
|
- created_at: '2020-12-05T17:48:50Z'
|
||||||
|
enc: |
|
||||||
|
-----BEGIN PGP MESSAGE-----
|
||||||
|
|
||||||
|
hQGMA/JorPHm1g9XAQv+L90aZXZRKICnvXOQxxhgSEJq/tudWNXt+HfC2/x4L/dO
|
||||||
|
pz6OXz/J20+x3mSOB8J9IkYXcy7NMr3LGYCX/BoupvngI9yGu0i2fj69AYLaQ4zi
|
||||||
|
a8wWH68YJX+FHeWIs8ltG2naYt2MLr/rbADaTpreILimywagXw6ViOwdVDRgh1JZ
|
||||||
|
qQtdDwEWUjD+t9Lve0s63Z6vFx8s6CnA+xqyB5ci84VMCqSmE4EvtN54JcFOzOHE
|
||||||
|
4CJJ2efYpSFZyvRTxwp33MyuZFGs8GF+HQfqy2EhLfmo+g2A5x0egahQptf3cGmi
|
||||||
|
3GcEIul9Oh/uPAd+LPB6zwlVbg849I8o8CbdDyjfHBt36pC54jmUg9YRPHTd+t0z
|
||||||
|
r+g17JXcCb5oBSUs6b5opKdtfH4nscYWFIffZ7LW7A5Tn8HLRcWYMW06+/N1Ft5j
|
||||||
|
TVqUutVreMXnuctXeGKS9/obpUqVpP8Lui5bYoJgCio7q4womxsR1spsNwtPa07u
|
||||||
|
G7knAzmYY73EI2AXLayf0l4BDkby4QPPpZKVR+kYeu7EFrMcMPqGl+IwLN03zHMM
|
||||||
|
xuCbSZ59zzsm/t6FnsfsVHXHILrVFcEH+2pkSAkRJHvBBmWK10KIdUh3Hgd238nf
|
||||||
|
GFv2i8CJiJCx8E8yAikD
|
||||||
|
=0gs7
|
||||||
|
-----END PGP MESSAGE-----
|
||||||
|
fp: C8F8A49D04A1AB639F8EA21CDBA4B1DCB1FA5BDD
|
||||||
|
encrypted_regex: ^(data|stringData)$
|
||||||
|
version: 3.6.1
|
||||||
Reference in New Issue
Block a user