Added prometheus alerts and slack configuration in separate config file. (4d381543) · Commits · MEC - Multi-access Edge Computing / ETSI MEC Sandbox

.meepctl-repocfg.yaml

+1 −1

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		version: 1.6.6
		version: 1.6.7
		repo:
		name: AdvantEDGE

charts/kube-prometheus-stack/templates/prometheus/additionalPrometheusRules.yaml

+1 −1

Original line number	Diff line number	Diff line
		{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}}
		{{- if and (.Values.alertmanager.enabled) (or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap) }}
		apiVersion: v1
		kind: List
		items:

charts/kube-prometheus-stack/templates/prometheus/pvc-deleter.yaml

+3 −2

Original line number	Diff line number	Diff line
		@@ -50,8 +50,9 @@ spec:
		args:
		- "delete"
		- "pvc"
		- "{{ .Values.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.metadata.name }}-prometheus-{{ template "kube-prometheus-stack.fullname" . }}-prometheus-0"
		- "{{ .Values.alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.metadata.name }}-alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager-0"
		- "--all"
		# - "{{ .Values.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.metadata.name }}-prometheus-{{ template "kube-prometheus-stack.fullname" . }}-prometheus-0"
		# - "{{ .Values.alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.metadata.name }}-alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager-0"
		serviceAccountName: pvc-deleter-sa
		securityContext:
		runAsUser: 0

charts/kube-prometheus-stack/values.yaml

+7 −42

Original line number	Diff line number	Diff line
		@@ -87,17 +87,6 @@ defaultRules:
		# - record: my_record
		# expr: 100 * my_record

		additionalPrometheusRulesMap: {}
		# - rule-name: CustomAlert
		# groups:
		# - name: CouchDBInstances
		# rules:
		# - alert: CouchDBDownAlert
		# expr: absent(couchdb_httpd_up) or couchdb_httpd_up < 1
		# for: 0m
		# annotations:
		# summary: CouchDB Node Down

		##
		global:
		rbac:
		@@ -154,47 +143,23 @@ alertmanager:
		## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file
		## https://prometheus.io/webtools/alerting/routing-tree-editor/
		##
		config:
		global:
		resolve_timeout: 5m
		route:
		group_by: ['job']
		group_wait: 30s
		group_interval: 5m
		repeat_interval: 12h
		receiver: 'null'
		routes:
		- match:
		alertname: Watchdog
		receiver: 'null'
		receivers:
		- name: 'null'

		# config:
		# global:
		# resolve_timeout: 5m
		# route:
		# group_by: [Alertname]
		# receiver: couchdb-alert
		# group_by: ['job']
		# group_wait: 30s
		# group_interval: 5m
		# repeat_interval: 12h
		# receiver: 'couchdb-alert'
		# receiver: 'null'
		# routes:
		# - match:
		# alertname: DeadMansSwitch
		# receiver: 'couchdb-alert'
		# alertname: Watchdog
		# receiver: 'null'
		# receivers:
		# - name: couchdb-alert
		# email_configs:
		# - to:
		# from:
		# smarthost:
		# auth_username:
		# auth_identity:
		# auth_password:

		# - name: 'null'
		# templates:
		# - '/etc/alertmanager/config/*.tmpl'

		## Pass the Alertmanager configuration directives through Helm's templating
		## engine. If the Alertmanager configuration contains Alertmanager templates,
		@@ -1515,7 +1480,7 @@ prometheus:
		# - thanos-gateway.domain.com

		ingress:
		enabled: true
		enabled: false

		# For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName
		# See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress

config/prometheus-alerts-config.yaml

0 → 100644

+124 −0

Original line number	Diff line number	Diff line
		additionalPrometheusRulesMap:
		- rule-name: PrometheusAlerts
		groups:
		- name: MEC Sandbox Alerts
		rules:
		- alert: NoOfActiveSessions
		expr: auth_svc_session_active >= 8
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Active sessions more than or equal to 8 (instance {{ $labels.instance }})
		description: Active seesions at MEC Sanbox >= 8\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- name: Node Alerts
		rules:
		- alert: HostRebooted
		expr: changes(node_boot_time_seconds[1d]) > 0
		for: 0m
		labels:
		severity: warning
		annotations:
		summary: Host rebooted (instance {{ $labels.instance }})
		description: Node just rebooted\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostOutOfMemory
		expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Host out of memory (instance {{ $labels.instance }})
		description: Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostOutOfDiskSpace
		expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Host out of disk space (instance {{ $labels.instance }})
		description: Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostMemoryUnderMemoryPressure
		expr: rate(node_vmstat_pgmajfault[1m]) > 1000
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Host memory under memory pressure (instance {{ $labels.instance }})
		description: The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostUnusualNetworkThroughputIn
		expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
		for: 5m
		labels:
		severity: warning
		annotations:
		summary: Host unusual network throughput in (instance {{ $labels.instance }})
		description: Host network interfaces are probably receiving too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostUnusualNetworkThroughputOut
		expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
		for: 5m
		labels:
		severity: warning
		annotations:
		summary: Host unusual network throughput out (instance {{ $labels.instance }})
		description: Host network interfaces are probably sending too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABEL = {{ $labels }}
		- alert: HostUnusualDiskReadRate
		expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
		for: 5m
		labels:
		severity: warning
		annotations:
		summary: Host unusual disk read rate (instance {{ $labels.instance }})
		description: Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostUnusualDiskWriteRate
		expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Host unusual disk write rate (instance {{ $labels.instance }})
		description: Disk is probably writing too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: HostHighCpuLoad
		expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
		for: 0m
		labels:
		severity: warning
		annotations:
		summary: Host high CPU load (instance {{ $labels.instance }})
		description: CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- name: Kubernetes Alerts
		rules:
		- alert: KubernetesPodCrashLooping
		expr: increase(kube_pod_container_status_restarts_total[1m]) > 3
		for: 2m
		labels:
		severity: warning
		annotations:
		summary: Kubernetes pod crash looping (instance {{ $labels.instance }})
		description: Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
		- alert: KubernetesPodNotHealthy
		expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending\|Unknown\|Failed"})[1h:]) > 0
		for: 0m
		labels:
		severity: critical
		annotations:
		summary: Kubernetes Pod not healthy (instance {{ $labels.instance }})
		description: Pod has been in a non-ready state for longer than an hour.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}

		alertmanager:
		enabled: false

		config:
		global:
		resolve_timeout: 5m
		slack_api_url: 'https://hooks.slack.com/services/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'
		route:
		group_by: [Alertname]
		receiver: slack-alerts
		group_wait: 30s
		group_interval: 5m
		repeat_interval: 24h
		receivers:
		- name: slack-alerts
		slack_configs:
		- channel: '#sandbox_alerts'
		send_resolved: true