Commit 27297f40 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/webui' into 'develop' 

Add Grafana Dashboards to WebUI

See merge request teraflow-h2020/controller!77
parents f5a1a716 4cb1a369

configure_dashboards.sh

0 → 100755
+42 −0
Original line number Diff line number Diff line 
#!/bin/bash



# Set the name of the Kubernetes namespace to deploy to.

K8S_NAMESPACE="tf-dev"

K8S_HOSTNAME="kubernetes-master"



export INFLUXDB_HOSTNAME=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')

export INFLUXDB_PORT=$(kubectl get service influx-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==8086)].nodePort}')

export INFLUXDB_USER=$(kubectl --namespace $K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_USER}' | base64 --decode)

export INFLUXDB_PASSWORD=$(kubectl --namespace $K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_PASSWORD}' | base64 --decode)

export INFLUXDB_DATABASE=$(kubectl --namespace $K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_DB}' | base64 --decode)

export INFLUXDB_URL="http://${INFLUXDB_HOSTNAME}:${INFLUXDB_PORT}"



export GRAFANA_HOSTNAME=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')

export GRAFANA_PORT=$(kubectl get service webuiservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}')

export GRAFANA_USERNAME="admin"

export GRAFANA_PASSWORD="admin123+"

export GRAFANA_URL="http://${GRAFANA_USERNAME}:${GRAFANA_PASSWORD}@${GRAFANA_HOSTNAME}:${GRAFANA_PORT}"



echo ${GRAFANA_URL}/api/datasources



# Create InfluxDB DataSource

# Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/

curl -X POST -H "Content-Type: application/json" -d '{

  "type"     : "influxdb",

  "name"     : "InfluxDB",

  "url"      : "'"$INFLUXDB_URL"'",

  "access"   : "proxy",

  "basicAuth": false,

  "user"     : "'"$INFLUXDB_USER"'",

  "password" : "'"$INFLUXDB_PASSWORD"'",

  "isDefault": true,

  "database" : "'"$INFLUXDB_DATABASE"'"

}' ${GRAFANA_URL}/api/datasources

echo



# Create Monitoring Dashboard

# Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/

curl -X POST -H "Content-Type: application/json" \

  -d '@src/webui/grafana_dashboard.json' \

  ${GRAFANA_URL}/api/dashboards/db

echo

manifests/webuiservice.yaml

+44 −0
Original line number Diff line number Diff line
@@ -12,6 +12,10 @@ spec: 
        app: webuiservice

    spec:

      terminationGracePeriodSeconds: 5

      securityContext:

        fsGroup: 472

        supplementalGroups:

          - 0

      containers:

      - name: server

        image: registry.gitlab.com/teraflow-h2020/controller/webui:latest
@@ -40,6 +44,41 @@ spec: 
          limits:

            cpu: 700m

            memory: 1024Mi

      - name: grafana

        image: grafana/grafana:8.2.6

        imagePullPolicy: IfNotPresent

        ports:

          - containerPort: 3000

            name: http-grafana

            protocol: TCP

        env:

        - name: GF_SECURITY_ADMIN_PASSWORD

          value: admin123+

        readinessProbe:

          failureThreshold: 3

          httpGet:

            path: /robots.txt

            port: 3000

            scheme: HTTP

          initialDelaySeconds: 5

          periodSeconds: 5

          successThreshold: 1

          timeoutSeconds: 2

        livenessProbe:

          failureThreshold: 3

          initialDelaySeconds: 5

          periodSeconds: 5

          successThreshold: 1

          tcpSocket:

            port: 3000

          timeoutSeconds: 1

        resources:

          requests:

            cpu: 250m

            memory: 750Mi

          limits:

            cpu: 700m

            memory: 1024Mi

---

apiVersion: v1

kind: Service
@@ -70,4 +109,9 @@ spec: 
    port: 8004

    targetPort: 8004

    nodePort: 30800

  - name: grafana

    protocol: TCP

    port: 3000

    targetPort: 3000

    nodePort: 30300

---

src/webui/genproto.sh

+3 −0
Original line number Diff line number Diff line
@@ -29,15 +29,18 @@ python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=p 
python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto device.proto

python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto service.proto

python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto monitoring.proto

python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto kpi_sample_types.proto



# rm proto/compute_pb2_grpc.py

rm proto/context_pb2_grpc.py

rm proto/device_pb2_grpc.py

rm proto/service_pb2_grpc.py

rm proto/monitoring_pb2_grpc.py

rm proto/kpi_sample_types_pb2_grpc.py



# sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/compute_pb2.py

sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/context_pb2.py

sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/device_pb2.py

sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/service_pb2.py

sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/monitoring_pb2.py

sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/kpi_sample_types_pb2.py

src/webui/grafana_dashboard.json

0 → 100644
+313 −0
Original line number Diff line number Diff line 
{"overwrite": true, "folderId": 0, "dashboard":

  {

    "id": null,

    "annotations": {

      "list": [

        {

          "builtIn": 1,

          "datasource": "-- Grafana --",

          "enable": true,

          "hide": true,

          "iconColor": "rgba(0, 211, 255, 1)",

          "name": "Annotations & Alerts",

          "target": {

            "limit": 100,

            "matchAny": false,

            "tags": [],

            "type": "dashboard"

          },

          "type": "dashboard"

        }

      ]

    },

    "editable": true,

    "fiscalYearStartMonth": 0,

    "gnetId": null,

    "graphTooltip": 0,

    "iteration": 1643919736138,

    "links": [],

    "liveNow": false,

    "panels": [

      {

        "datasource": null,

        "fieldConfig": {

          "defaults": {

            "color": {

              "mode": "palette-classic"

            },

            "custom": {

              "axisLabel": "",

              "axisPlacement": "auto",

              "barAlignment": 0,

              "drawStyle": "line",

              "fillOpacity": 0,

              "gradientMode": "none",

              "hideFrom": {

                "legend": false,

                "tooltip": false,

                "viz": false

              },

              "lineInterpolation": "smooth",

              "lineWidth": 1,

              "pointSize": 5,

              "scaleDistribution": {

                "type": "linear"

              },

              "showPoints": "always",

              "spanNulls": false,

              "stacking": {

                "group": "A",

                "mode": "none"

              },

              "thresholdsStyle": {

                "mode": "off"

              }

            },

            "mappings": [],

            "thresholds": {

              "mode": "absolute",

              "steps": [

                {

                  "color": "green",

                  "value": null

                },

                {

                  "color": "red",

                  "value": 80

                }

              ]

            }

          },

          "overrides": [

            {

              "matcher": {

                "id": "byRegexp",

                "options": ".* PACKETS_.*"

              },

              "properties": [

                {

                  "id": "custom.axisPlacement",

                  "value": "left"

                },

                {

                  "id": "unit",

                  "value": "short"

                },

                {

                  "id": "custom.axisLabel",

                  "value": "Packets"

                }

              ]

            },

            {

              "matcher": {

                "id": "byRegexp",

                "options": ".* BYTES_.*"

              },

              "properties": [

                {

                  "id": "custom.axisPlacement",

                  "value": "right"

                },

                {

                  "id": "unit",

                  "value": "decbytes"

                },

                {

                  "id": "custom.axisLabel",

                  "value": "Bytes"

                }

              ]

            }

          ]

        },

        "gridPos": {

          "h": 19,

          "w": 24,

          "x": 0,

          "y": 0

        },

        "id": 2,

        "options": {

          "legend": {

            "calcs": [

              "first",

              "min",

              "mean",

              "max",

              "lastNotNull"

            ],

            "displayMode": "table",

            "placement": "right"

          },

          "tooltip": {

            "mode": "multi"

          }

        },

        "targets": [

          {

            "alias": "$tag_device_id $tag_endpoint_id $tag_kpi_sample_type",

            "groupBy": [

              {

                "params": [

                  "device_id"

                ],

                "type": "tag"

              },

              {

                "params": [

                  "endpoint_id"

                ],

                "type": "tag"

              },

              {

                "params": [

                  "kpi_sample_type"

                ],

                "type": "tag"

              }

            ],

            "measurement": "samples",

            "orderByTime": "ASC",

            "policy": "autogen",

            "refId": "A",

            "resultFormat": "time_series",

            "select": [

              [

                {

                  "params": [

                    "kpi_value"

                  ],

                  "type": "field"

                }

              ]

            ],

            "tags": [

              {

                "key": "device_id",

                "operator": "=~",

                "value": "/^$device_id$/"

              },

              {

                "condition": "AND",

                "key": "endpoint_id",

                "operator": "=~",

                "value": "/^$endpoint_id$/"

              },

              {

                "condition": "AND",

                "key": "kpi_sample_type",

                "operator": "=~",

                "value": "/^$kpi_sample_type$/"

              }

            ]

          }

        ],

        "title": "L3 Monitoring Packets/Bytes Received/Sent",

        "transformations": [],

        "type": "timeseries"

      }

    ],

    "refresh": "5s",

    "schemaVersion": 32,

    "style": "dark",

    "tags": [],

    "templating": {

      "list": [

        {

          "allValue": null,

          "current": {

            "selected": true,

            "text": [

              "R1-INF"

            ],

            "value": [

              "R1-INF"

            ]

          },

          "datasource": null,

          "definition": "SHOW TAG VALUES FROM samples WITH KEY=\"device_id\"",

          "description": null,

          "error": null,

          "hide": 0,

          "includeAll": true,

          "label": "Device",

          "multi": true,

          "name": "device_id",

          "options": [],

          "query": "SHOW TAG VALUES FROM samples WITH KEY=\"device_id\"",

          "refresh": 2,

          "regex": "",

          "skipUrlSync": false,

          "sort": 0,

          "type": "query"

        },

        {

          "allValue": null,

          "current": {

            "selected": true,

            "text": [

              "EP100"

            ],

            "value": [

              "EP100"

            ]

          },

          "datasource": null,

          "definition": "SHOW TAG VALUES FROM samples WITH KEY=\"endpoint_id\" WHERE \"device_id\"=~/^$device_id$/",

          "description": null,

          "error": null,

          "hide": 0,

          "includeAll": true,

          "label": "EndPoint",

          "multi": true,

          "name": "endpoint_id",

          "options": [],

          "query": "SHOW TAG VALUES FROM samples WITH KEY=\"endpoint_id\" WHERE \"device_id\"=~/^$device_id$/",

          "refresh": 2,

          "regex": "",

          "skipUrlSync": false,

          "sort": 0,

          "type": "query"

        },

        {

          "allValue": null,

          "current": {

            "selected": true,

            "text": [

              "All"

            ],

            "value": [

              "$__all"

            ]

          },

          "datasource": null,

          "definition": "SHOW TAG VALUES FROM samples WITH KEY=\"kpi_sample_type\"",

          "description": null,

          "error": null,

          "hide": 0,

          "includeAll": true,

          "label": "Kpi Sample Type",

          "multi": true,

          "name": "kpi_sample_type",

          "options": [],

          "query": "SHOW TAG VALUES FROM samples WITH KEY=\"kpi_sample_type\"",

          "refresh": 2,

          "regex": "",

          "skipUrlSync": false,

          "sort": 0,

          "type": "query"

        }

      ]

    },

    "time": {

      "from": "now-5m",

      "to": "now"

    },

    "timepicker": {},

    "timezone": "",

    "title": "L3 Monitoring",

    "uid": "5d_I5W-nz",

    "version": 2

  }

}

src/webui/proto/context_pb2.py

+338 −211

File changed.

Preview size limit exceeded, changes collapsed.