From 08d4197db6d5d9b4e68dadbd4c6b9b3fce97d4a2 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Tue, 28 Mar 2023 13:50:16 +0200 Subject: [PATCH 1/4] Changes to the deploy and webui yaml to accommodate the new metrics collection framework --- deploy/tfs.sh | 18 ++++-- manifests/nginx_ingress_http_new.yaml | 81 +++++++++++++++++++++++++++ manifests/webuiservice.yaml | 37 ------------ 3 files changed, 93 insertions(+), 43 deletions(-) create mode 100644 manifests/nginx_ingress_http_new.yaml diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 4c6dc95d2..a9dc74ef0 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -241,7 +241,8 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Adapting '$COMPONENT' manifest file..." MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml" - cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST" + # cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST" + cat ./manifests/"${COMPONENT}"service.yaml | linkerd inject - --proxy-cpu-request "10m" --proxy-cpu-limit "1" --proxy-memory-request "64Mi" --proxy-memory-limit "256Mi" > "$MANIFEST" if [ "$COMPONENT" == "pathcomp" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') @@ -328,14 +329,19 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]]; then echo "Configuring WebUI DataStores and Dashboards..." sleep 5 - INGRESS_CTRL_NAME=$(echo "${TFS_K8S_NAMESPACE}" | sed "s/tfs/nginx-ingress-microk8s-controller/g") - EXT_HTTP_PORT=$(kubectl get daemonsets.apps --namespace ingress ${INGRESS_CTRL_NAME} \ - -o 'jsonpath={.spec.template.spec.containers[?(@.name=="nginx-ingress-microk8s")].ports[?(@.name=="http")].hostPort}') + # INGRESS_CTRL_NAME=$(echo "${TFS_K8S_NAMESPACE}" | sed "s/tfs/nginx-ingress-microk8s-controller/g") + # EXT_HTTP_PORT=$(kubectl get daemonsets.apps --namespace ingress ${INGRESS_CTRL_NAME} \ + # -o 'jsonpath={.spec.template.spec.containers[?(@.name=="nginx-ingress-microk8s")].ports[?(@.name=="http")].hostPort}') + + # get IP and port of grafana in the `monitoring` namespace + GRAFANA_IP=$(kubectl get service/grafana -n monitoring -o jsonpath='{.spec.clusterIP}') + GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') # Exposed through the ingress controller "tfs-ingress" - GRAFANA_URL="127.0.0.1:${EXT_HTTP_PORT}/grafana" + # GRAFANA_URL="127.0.0.1:${EXT_HTTP_PORT}/grafana" + GRAFANA_URL="${GRAFANA_IP}:${GRAFANA_PORT}" - # Default Grafana credentials + # Default Grafana credentials when installed with the `monitoring` addon GRAFANA_USERNAME="admin" GRAFANA_PASSWORD="admin" diff --git a/manifests/nginx_ingress_http_new.yaml b/manifests/nginx_ingress_http_new.yaml new file mode 100644 index 000000000..e9a389ccc --- /dev/null +++ b/manifests/nginx_ingress_http_new.yaml @@ -0,0 +1,81 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tfs-ingress + namespace: tfs + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + rules: + - http: + paths: + - path: /webui(/|$)(.*) + pathType: Prefix + backend: + service: + name: webuiservice + port: + number: 8004 + - path: /grafana(/|$)(.*) + pathType: Prefix + backend: + service: + name: webuiservice + port: + number: 3000 + #- path: /context(/|$)(.*) + # pathType: Prefix + # backend: + # service: + # name: contextservice + # port: + # number: 8080 + - path: /()(restconf/.*) + pathType: Prefix + backend: + service: + name: computeservice + port: + number: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + namespace: monitoring + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/proxy-redirect-from: "/" + nginx.ingress.kubernetes.io/proxy-redirect-to: "/newgrafana/" +spec: + rules: + - http: + paths: + - path: /newgrafana(/|$)(.*) + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 + #- path: /context(/|$)(.*) + # pathType: Prefix + # backend: + # service: + # name: contextservice + # port: + # number: 8080 diff --git a/manifests/webuiservice.yaml b/manifests/webuiservice.yaml index 234075f73..255ee68a2 100644 --- a/manifests/webuiservice.yaml +++ b/manifests/webuiservice.yaml @@ -61,43 +61,6 @@ spec: limits: cpu: 500m memory: 512Mi - - name: grafana - image: grafana/grafana:8.5.11 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3000 - name: http-grafana - protocol: TCP - env: - - name: GF_SERVER_ROOT_URL - value: "http://0.0.0.0:3000/grafana/" - - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: "true" - readinessProbe: - failureThreshold: 3 - httpGet: - path: /robots.txt - port: 3000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 2 - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - tcpSocket: - port: 3000 - timeoutSeconds: 1 - resources: - requests: - cpu: 150m - memory: 512Mi - limits: - cpu: 500m - memory: 1024Mi --- apiVersion: v1 kind: Service -- GitLab From dadfe9186e9710adfdddbf3105f8489a1cfefbd0 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Tue, 28 Mar 2023 13:52:50 +0200 Subject: [PATCH 2/4] Removing unnecessary files --- manifests/nginx_ingress_http_new.yaml | 81 --------------------------- 1 file changed, 81 deletions(-) delete mode 100644 manifests/nginx_ingress_http_new.yaml diff --git a/manifests/nginx_ingress_http_new.yaml b/manifests/nginx_ingress_http_new.yaml deleted file mode 100644 index e9a389ccc..000000000 --- a/manifests/nginx_ingress_http_new.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: tfs-ingress - namespace: tfs - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$2 -spec: - rules: - - http: - paths: - - path: /webui(/|$)(.*) - pathType: Prefix - backend: - service: - name: webuiservice - port: - number: 8004 - - path: /grafana(/|$)(.*) - pathType: Prefix - backend: - service: - name: webuiservice - port: - number: 3000 - #- path: /context(/|$)(.*) - # pathType: Prefix - # backend: - # service: - # name: contextservice - # port: - # number: 8080 - - path: /()(restconf/.*) - pathType: Prefix - backend: - service: - name: computeservice - port: - number: 8080 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: grafana-ingress - namespace: monitoring - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/proxy-redirect-from: "/" - nginx.ingress.kubernetes.io/proxy-redirect-to: "/newgrafana/" -spec: - rules: - - http: - paths: - - path: /newgrafana(/|$)(.*) - pathType: Prefix - backend: - service: - name: grafana - port: - number: 3000 - #- path: /context(/|$)(.*) - # pathType: Prefix - # backend: - # service: - # name: contextservice - # port: - # number: 8080 -- GitLab From 21859f8d7acc95649f557657c5f85eefb5d03508 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Tue, 28 Mar 2023 13:58:08 +0200 Subject: [PATCH 3/4] Including a script to open the dashboard. --- scripts/open_dashboard.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/open_dashboard.sh diff --git a/scripts/open_dashboard.sh b/scripts/open_dashboard.sh new file mode 100755 index 000000000..2ff15684a --- /dev/null +++ b/scripts/open_dashboard.sh @@ -0,0 +1,27 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#/bin/bash + +# this script opens the dashboard + +GRAFANA_IP=$(kubectl get service/grafana -n monitoring -o jsonpath='{.spec.clusterIP}') +GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') +URL=http://${GRAFANA_IP}:${GRAFANA_PORT} + +echo Opening Dashboard on URL ${URL} + +# curl -kL ${URL} + +python3 -m webbrowser ${URL} -- GitLab From 2093df02db66ee52e14237e648632f8d6fdc3211 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 30 Mar 2023 08:10:20 +0000 Subject: [PATCH 4/4] Deploy scripts: - Added script to expose observability framework - Extended TFS deploy scripts to run automatically run expose observability scripts - Adapted TFS deploy script - Script cleanup --- deploy/all.sh | 12 ++++++++ deploy/expose_dashboard.sh | 58 +++++++++++++++++++++++++++++++++++ deploy/tfs.sh | 13 ++++++-- my_deploy.sh | 9 ++++++ scripts/old/open_dashboard.sh | 4 +-- scripts/open_dashboard.sh | 27 ---------------- 6 files changed, 91 insertions(+), 32 deletions(-) create mode 100755 deploy/expose_dashboard.sh mode change 100755 => 100644 scripts/old/open_dashboard.sh delete mode 100755 scripts/open_dashboard.sh diff --git a/deploy/all.sh b/deploy/all.sh index 6f8331b76..9584dd32d 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -147,6 +147,15 @@ export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""} export QDB_REDEPLOY=${QDB_REDEPLOY:-""} +# ----- K8s Observability ------------------------------------------------------ + +# If not already set, set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"} + +# If not already set, set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} + + ######################################################################################################################## # Automated steps start here ######################################################################################################################## @@ -160,6 +169,9 @@ export QDB_REDEPLOY=${QDB_REDEPLOY:-""} # Deploy QuestDB ./deploy/qdb.sh +# Expose Dashboard +./deploy/expose_dashboard.sh + # Deploy TeraFlowSDN ./deploy/tfs.sh diff --git a/deploy/expose_dashboard.sh b/deploy/expose_dashboard.sh new file mode 100755 index 000000000..60b41c7b7 --- /dev/null +++ b/deploy/expose_dashboard.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +######################################################################################################################## +# Read deployment settings +######################################################################################################################## + +# If not already set, set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"} + +# If not already set, set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} + + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +function expose_dashboard() { + echo "Prometheus Port Mapping" + echo ">>> Expose Prometheus HTTP Mgmt GUI port (9090->${PROM_EXT_PORT_HTTP})" + PROM_PORT_HTTP=$(kubectl --namespace monitoring get service prometheus-k8s -o 'jsonpath={.spec.ports[?(@.name=="web")].port}') + PATCH='{"data": {"'${PROM_EXT_PORT_HTTP}'": "monitoring/prometheus-k8s:'${PROM_PORT_HTTP}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${PROM_EXT_PORT_HTTP}', "hostPort": '${PROM_EXT_PORT_HTTP}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo + + echo "Grafana Port Mapping" + echo ">>> Expose Grafana HTTP Mgmt GUI port (3000->${GRAF_EXT_PORT_HTTP})" + GRAF_PORT_HTTP=$(kubectl --namespace monitoring get service grafana -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') + PATCH='{"data": {"'${GRAF_EXT_PORT_HTTP}'": "monitoring/grafana:'${GRAF_PORT_HTTP}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${GRAF_EXT_PORT_HTTP}', "hostPort": '${GRAF_EXT_PORT_HTTP}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo +} + +expose_dashboard diff --git a/deploy/tfs.sh b/deploy/tfs.sh index a9dc74ef0..905cce80e 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -106,6 +106,15 @@ export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kp export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"} +# ----- K8s Observability ------------------------------------------------------ + +# If not already set, set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"} + +# If not already set, set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} + + ######################################################################################################################## # Automated steps start here ######################################################################################################################## @@ -335,11 +344,11 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]]; then # get IP and port of grafana in the `monitoring` namespace GRAFANA_IP=$(kubectl get service/grafana -n monitoring -o jsonpath='{.spec.clusterIP}') - GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') + #GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') # Exposed through the ingress controller "tfs-ingress" # GRAFANA_URL="127.0.0.1:${EXT_HTTP_PORT}/grafana" - GRAFANA_URL="${GRAFANA_IP}:${GRAFANA_PORT}" + GRAFANA_URL="${GRAFANA_IP}:${GRAF_EXT_PORT_HTTP}" # Default Grafana credentials when installed with the `monitoring` addon GRAFANA_USERNAME="admin" diff --git a/my_deploy.sh b/my_deploy.sh index 22a7ae815..f561b1ea8 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -115,3 +115,12 @@ export QDB_DROP_TABLES_IF_EXIST="" # Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/scripts/old/open_dashboard.sh b/scripts/old/open_dashboard.sh old mode 100755 new mode 100644 index 4ea206f45..2ff15684a --- a/scripts/old/open_dashboard.sh +++ b/scripts/old/open_dashboard.sh @@ -16,9 +16,7 @@ # this script opens the dashboard -K8S_NAMESPACE=${K8S_NAMESPACE:-'tfs'} - -GRAFANA_IP=$(kubectl get service/webuiservice -n ${TFS_K8S_NAMESPACE} -o jsonpath='{.spec.clusterIP}') +GRAFANA_IP=$(kubectl get service/grafana -n monitoring -o jsonpath='{.spec.clusterIP}') GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') URL=http://${GRAFANA_IP}:${GRAFANA_PORT} diff --git a/scripts/open_dashboard.sh b/scripts/open_dashboard.sh deleted file mode 100755 index 2ff15684a..000000000 --- a/scripts/open_dashboard.sh +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#/bin/bash - -# this script opens the dashboard - -GRAFANA_IP=$(kubectl get service/grafana -n monitoring -o jsonpath='{.spec.clusterIP}') -GRAFANA_PORT=3000 #$(kubectl get service webuiservice --namespace $TFS_K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==3000)].nodePort}') -URL=http://${GRAFANA_IP}:${GRAFANA_PORT} - -echo Opening Dashboard on URL ${URL} - -# curl -kL ${URL} - -python3 -m webbrowser ${URL} -- GitLab