Commit c2a65dc4 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/249-cttc-telemetry-enhancement' into 'develop' 

Resolve "(CTTC) Telemetry Enhancement"

See merge request !320
parents 6573fa12 17228dd5

.gitignore

+0 −2
Original line number Diff line number Diff line
@@ -179,5 +179,3 @@ libyang/ 
# Other logs

**/logs/*.log.*

# PySpark checkpoints

src/analytics/.spark/*

deploy/all.sh

+3 −0
Original line number Diff line number Diff line
@@ -215,6 +215,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} 
# Deploy Apache Kafka

./deploy/kafka.sh



#Deploy Monitoring (Prometheus, Mimir, Grafana)

./deploy/monitoring.sh



# Expose Dashboard

./deploy/expose_dashboard.sh

deploy/monitoring.sh

0 → 100755
+132 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



set -euo pipefail



# -----------------------------------------------------------

# Global namespace for all deployments

# -----------------------------------------------------------

NAMESPACE="monitoring"

VALUES_FILE_PATH="manifests/monitoring"



# -----------------------------------------------------------

# Prometheus Configuration

# -----------------------------------------------------------

RELEASE_NAME_PROM="mon-prometheus"

CHART_REPO_NAME_PROM="prometheus-community"

CHART_REPO_URL_PROM="https://prometheus-community.github.io/helm-charts"

CHART_NAME_PROM="prometheus"

VALUES_FILE_PROM="$VALUES_FILE_PATH/prometheus_values.yaml"



# -----------------------------------------------------------

# Mimir Configuration

# -----------------------------------------------------------

# RELEASE_NAME_MIMIR="mon-mimir"

# CHART_REPO_NAME_MIMIR="grafana"

# CHART_REPO_URL_MIMIR="https://grafana.github.io/helm-charts"

# CHART_NAME_MIMIR="mimir-distributed"

# VALUES_FILE_MIMIR="$VALUES_FILE_PATH/mimir_values.yaml"



# -----------------------------------------------------------

# Grafana Configuration

# -----------------------------------------------------------

# RELEASE_NAME_GRAFANA="mon-grafana"

# CHART_REPO_NAME_GRAFANA="grafana"

# CHART_REPO_URL_GRAFANA="https://grafana.github.io/helm-charts"

# CHART_NAME_GRAFANA="grafana"

# VALUES_FILE_GRAFANA="$VALUES_FILE_PATH/grafana_values.yaml"





# -----------------------------------------------------------

# Function to deploy or upgrade a Helm chart

# -----------------------------------------------------------

deploy_chart() {

  local release_name="$1"

  local chart_repo_name="$2"

  local chart_repo_url="$3"

  local chart_name="$4"

  local values_file="$5"

  local namespace="$6"



  echo ">>> Deploying [${release_name}] from repo [${chart_repo_name}]..."



  # Add or update the Helm repo

  echo "Adding/updating Helm repo: $chart_repo_name -> $chart_repo_url"

  helm repo add "$chart_repo_name" "$chart_repo_url" || true

  helm repo update



  # Create namespace if needed

  echo "Creating namespace '$namespace' if it doesn't exist..."

  kubectl get namespace "$namespace" >/dev/null 2>&1 || kubectl create namespace "$namespace"



  # Install or upgrade the chart

  if [ -n "$values_file" ] && [ -f "$values_file" ]; then

    echo "Installing/Upgrading $release_name using custom values from $values_file..."

    helm upgrade --install "$release_name" "$chart_repo_name/$chart_name" \

      --namespace "$namespace" \

      --values "$values_file"

  else

    echo "Installing/Upgrading $release_name with default chart values..."

    helm upgrade --install "$release_name" "$chart_repo_name/$chart_name" \

      --namespace "$namespace"

  fi



  echo "<<< Deployment initiated for [$release_name]."

  echo

}





# -----------------------------------------------------------

# Actual Deployments

# -----------------------------------------------------------



# 1) Deploy Prometheus

deploy_chart "$RELEASE_NAME_PROM" \

             "$CHART_REPO_NAME_PROM" \

             "$CHART_REPO_URL_PROM" \

             "$CHART_NAME_PROM" \

             "$VALUES_FILE_PROM" \

             "$NAMESPACE"



# Optionally wait for Prometheus server pod to become ready

kubectl rollout status deployment/"$RELEASE_NAME_PROM-server" -n "$NAMESPACE" || true





# 2) Deploy Mimir

# deploy_chart "$RELEASE_NAME_MIMIR" \

#              "$CHART_REPO_NAME_MIMIR" \

#              "$CHART_REPO_URL_MIMIR" \

#              "$CHART_NAME_MIMIR" \

#              "$VALUES_FILE_MIMIR" \

#              "$NAMESPACE"



# Depending on how Mimir runs (StatefulSets, Deployments), you can wait for

# the correct resource to be ready. For example:

# kubectl rollout status statefulset/"$RELEASE_NAME_MIMIR-distributor" -n "$NAMESPACE" || true





# 3) Deploy Grafana

# deploy_chart "$RELEASE_NAME_GRAFANA" \

#              "$CHART_REPO_NAME_GRAFANA" \

#              "$CHART_REPO_URL_GRAFANA" \

#              "$CHART_NAME_GRAFANA" \

#              "$VALUES_FILE_GRAFANA" \

#              "$NAMESPACE"



# kubectl rollout status deployment/"$RELEASE_NAME_GRAFANA" -n "$NAMESPACE" || true



# -----------------------------------------------------------

echo "All deployments completed!"

manifests/kpi_value_writerservice.yaml

+2 −0
Original line number Diff line number Diff line
@@ -39,6 +39,8 @@ spec: 
          env:

            - name: LOG_LEVEL

              value: "INFO"

            - name: PUSHGATEWAY_URL

              value: "http://mon-prometheus-prometheus-pushgateway.monitoring.svc.cluster.local:9091"

          envFrom:

            - secretRef:

                name: kfk-kpi-data

manifests/monitoring/grafana_values.yaml

0 → 100644
+235 −0
Original line number Diff line number Diff line 
rbac:

  create: true

  ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true)

  # useExistingRole: name-of-some-role

  # useExistingClusterRole: name-of-some-clusterRole

  pspEnabled: false

  pspUseAppArmor: false

  namespaced: false



serviceAccount:

  create: true

  name:

  nameTest:

  ## ServiceAccount labels.

  automountServiceAccountToken: false



replicas: 1



## Create a headless service for the deployment

headlessService: false



## Should the service account be auto mounted on the pod

automountServiceAccountToken: true



## Create HorizontalPodAutoscaler object for deployment type

#

autoscaling:

  enabled: false

  minReplicas: 1

  maxReplicas: 3

  targetCPU: "60"

  targetMemory: ""

  behavior: {}



deploymentStrategy:

  type: RollingUpdate



readinessProbe:

  httpGet:

    path: /api/health

    port: 3000



livenessProbe:

  httpGet:

    path: /api/health

    port: 3000

  initialDelaySeconds: 60

  timeoutSeconds: 30

  failureThreshold: 10



image:

  registry: docker.io

  repository: grafana/grafana

  # Overrides the Grafana image tag whose default is the chart appVersion

  tag: ""

  sha: ""

  pullPolicy: IfNotPresent



  ## Optionally specify an array of imagePullSecrets.

  ## Secrets must be manually created in the namespace.

  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/

  ## Can be templated.

  ##

  pullSecrets: []

  #   - myRegistrKeySecretName



testFramework:

  enabled: true

  ## The type of Helm hook used to run this test. Defaults to test.

  ## ref: https://helm.sh/docs/topics/charts_hooks/#the-available-hooks

  ##

  # hookType: test

  image:

    # -- The Docker registry

    registry: docker.io

    repository: bats/bats

    tag: "v1.4.1"

  imagePullPolicy: IfNotPresent



# dns configuration for pod

dnsPolicy: ~

dnsConfig: {}

  # nameservers:

  #   - 8.8.8.8

  #   options:

  #   - name: ndots

  #     value: "2"

  #   - name: edns0



securityContext:

  runAsNonRoot: true

  runAsUser: 472

  runAsGroup: 472

  fsGroup: 472



containerSecurityContext:

  allowPrivilegeEscalation: false

  capabilities:

    drop:

    - ALL

  seccompProfile:

    type: RuntimeDefault



# Enable creating the grafana configmap

createConfigmap: true



downloadDashboardsImage:

  registry: docker.io

  repository: curlimages/curl

  tag: 8.9.1

  sha: ""

  pullPolicy: IfNotPresent



downloadDashboards:

  env: {}

  envFromSecret: ""

  resources: {}

  securityContext:

    allowPrivilegeEscalation: false

    capabilities:

      drop:

      - ALL

    seccompProfile:

      type: RuntimeDefault

  envValueFrom: {}

  #  ENV_NAME:

  #    configMapKeyRef:

  #      name: configmap-name

  #      key: value_key



## Pod Annotations

# podAnnotations: {}



## ConfigMap Annotations

# configMapAnnotations: {}

  # argocd.argoproj.io/sync-options: Replace=true



## Pod Labels

# podLabels: {}



podPortName: grafana

gossipPortName: gossip

## Deployment annotations

# annotations: {}



service:

  enabled: true

  type: NodePort

  port: 80

  targetPort: 3000

  nodePort: 30080

  portName: service



## Enable persistence using Persistent Volume Claims

## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/

##

persistence:

  type: pvc

  enabled: true

  # storageClassName: default

  accessModes:

    - ReadWriteOnce

  size: 10Gi

  # annotations: {}

  finalizers:

    - kubernetes.io/pvc-protection



  disableWarning: false



  ## If 'lookupVolumeName' is set to true, Helm will attempt to retrieve

  ## the current value of 'spec.volumeName' and incorporate it into the template.

  lookupVolumeName: true



# Administrator credentials when not using an existing secret (see below)

adminUser: admin

# adminPassword: strongpassword



# Use an existing secret for the admin user.

admin:

  ## Name of the secret. Can be templated.

  existingSecret: ""

  userKey: admin-user

  passwordKey: admin-password



## Configure grafana datasources

## ref: http://docs.grafana.org/administration/provisioning/#datasources

##

datasources: 

 datasources.yaml:

   apiVersion: 1

   datasources:

   - name: Prometheus

     type: prometheus

     url: http://mon-prometheus-server.monitoring.svc.cluster.local

     access: proxy

     isDefault: true

   - name: Mimir

     type: prometheus

     url: http://mimir-nginx.mon-mimir.svc:80/prometheus

     access: proxy

     isDefault: false



## Grafana's primary configuration

## NOTE: values in map will be converted to ini format

## ref: http://docs.grafana.org/installation/configuration/

##

grafana.ini:

  paths:

    data: /var/lib/grafana/

    logs: /var/log/grafana

    plugins: /var/lib/grafana/plugins

    provisioning: /etc/grafana/provisioning

  analytics:

    check_for_updates: true

  log:

    mode: console

  grafana_net:

    url: https://grafana.net

  server:

    domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ tpl (.Values.ingress.hosts | first) . }}{{ else }}''{{ end }}"



## Number of old ReplicaSets to retain

##

revisionHistoryLimit: 5



# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret

# values are not exposed in the rendered grafana.ini configmap. It is enabled by default.

#

# To pass values into grafana.ini without exposing them in a configmap, use variable expansion:

# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion

#

# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap,

# you can disable this check by setting assertNoLeakedSecrets to false.

assertNoLeakedSecrets: true