diff --git a/deploy/all.sh b/deploy/all.sh index a99607f5b907c2bd1e1b4b889bef881874a63967..09239afed7ba036b214742e636017a58c072f6b3 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -25,14 +25,14 @@ # By default, assume internal MicroK8s registry is used. export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"} -# If not already set, set the list of components you want to build images for, and deploy. +# If not already set, set the list of components, separated by spaces, you want to build images for, and deploy. # By default, only basic components are deployed -export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"} +export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device automation monitoring pathcomp service slice compute webui load_generator"} # If not already set, set the tag you want to use for your images. export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"} -# If not already set, set the name of the Kubernetes namespace to deploy to. +# If not already set, set the name of the Kubernetes namespace to deploy TFS to. export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} # If not already set, set additional manifest files to be applied after the deployment @@ -41,7 +41,7 @@ export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""} # If not already set, set the new Grafana admin password export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"} -# If not already set, disable skip-build flag. +# If not already set, disable skip-build flag to rebuild the Docker images. # If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used. export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""} @@ -60,12 +60,6 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} # If not already set, set the database name to be used by Context. export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"} -# If not already set, set the name of the secret where CockroachDB data and credentials will be stored. -export CRDB_SECRET_NAME=${CRDB_SECRET_NAME:-"crdb-data"} - -# If not already set, set the namespace where the secret containing CockroachDB data and credentials will be stored. -export CRDB_SECRET_NAMESPACE=${CRDB_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}} - # If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'. # "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while # checking/deploying CockroachDB. @@ -78,7 +72,7 @@ export CRDB_SECRET_NAMESPACE=${CRDB_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}} # Ref: https://www.cockroachlabs.com/docs/stable/recommended-production-settings.html export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"} -# If not already set, disable flag for dropping database if exists. +# If not already set, disable flag for dropping database, if it exists. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION! # If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while # checking/deploying CockroachDB. @@ -96,12 +90,6 @@ export CRDB_REDEPLOY=${CRDB_REDEPLOY:-""} # If not already set, set the namespace where NATS will be deployed. export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"} -# If not already set, set the name of the secret where NATS data and credentials will be stored. -export NATS_SECRET_NAME=${NATS_SECRET_NAME:-"nats-data"} - -# If not already set, set the namespace where the secret containing NATS data and credentials will be stored. -export NATS_SECRET_NAMESPACE=${NATS_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}} - # If not already set, disable flag for re-deploying NATS from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! # If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS. @@ -113,20 +101,20 @@ export NATS_REDEPLOY=${NATS_REDEPLOY:-""} # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} -# If not already set, set the database username to be used by Monitoring. +# If not already set, set the database username to be used for QuestDB. export QDB_USERNAME=${QDB_USERNAME:-"admin"} -# If not already set, set the database user's password to be used by Monitoring. +# If not already set, set the database user's password to be used for QuestDB. export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"} +# If not already set, set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"} -## If not already set, disable flag for dropping table if exists. -## WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION! -## If QDB_DROP_TABLE_IF_EXISTS is "YES", the table pointed by variable QDB_TABLE will be dropped while -## checking/deploying QuestDB. -#export QDB_DROP_TABLE_IF_EXISTS=${QDB_DROP_TABLE_IF_EXISTS:-""} +# If not already set, disable flag for dropping tables if they exist. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION! +# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable +# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB. +export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""} # If not already set, disable flag for re-deploying QuestDB from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION! diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 98d011f190196b803be27200b8bc348b30c87055..4e8cfe2c399fb0e943c90e5c585f93f0707ca835 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -66,9 +66,6 @@ CRDB_MANIFESTS_PATH="manifests/cockroachdb" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" mkdir -p $TMP_MANIFESTS_FOLDER -TMP_LOGS_FOLDER="$TMP_FOLDER/logs" -mkdir -p $TMP_LOGS_FOLDER -CRDB_LOG_FILE="$TMP_LOGS_FOLDER/crdb_deploy.log" function crdb_deploy_single() { echo "CockroachDB Namespace" diff --git a/deploy/nats.sh b/deploy/nats.sh index 115a185302236b80db385212cd772100392329af..9edbc7765a09135d62a6021c5f2b0669e36a69a4 100755 --- a/deploy/nats.sh +++ b/deploy/nats.sh @@ -31,14 +31,6 @@ export NATS_REDEPLOY=${NATS_REDEPLOY:-""} # Automated steps start here ######################################################################################################################## -# Constants -TMP_FOLDER="./tmp" -NATS_MANIFESTS_PATH="manifests/nats" - -# Create a tmp folder for files modified during the deployment -TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" -mkdir -p $TMP_MANIFESTS_FOLDER - function nats_deploy_single() { echo "NATS Namespace" echo ">>> Create NATS Namespace (if missing)" diff --git a/deploy/qdb.sh b/deploy/qdb.sh index d9a4de353b3309ef0a8a34310089e9bff31589fa..a654088049df871fac0f4d19c225b2246f464f8e 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -21,20 +21,20 @@ # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} -# If not already set, set the database username to be used by Monitoring. +# If not already set, set the database username to be used for QuestDB. export QDB_USERNAME=${QDB_USERNAME:-"admin"} -# If not already set, set the database user's password to be used by Monitoring. +# If not already set, set the database user's password to be used for QuestDB. export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"} +# If not already set, set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"} -## If not already set, disable flag for dropping table if exists. -## WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION! -## If QDB_DROP_TABLE_IF_EXISTS is "YES", the table pointed by variable QDB_TABLE will be dropped while -## checking/deploying QuestDB. -#export QDB_DROP_TABLE_IF_EXISTS=${QDB_DROP_TABLE_IF_EXISTS:-""} +# If not already set, disable flag for dropping tables if they exist. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION! +# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable +# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB. +export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""} # If not already set, disable flag for re-deploying QuestDB from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION! @@ -52,9 +52,6 @@ QDB_MANIFESTS_PATH="manifests/questdb" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" -TMP_LOGS_FOLDER="$TMP_FOLDER/logs" -QDB_LOG_FILE="$TMP_LOGS_FOLDER/qdb_deploy.log" -mkdir -p $TMP_LOGS_FOLDER function qdb_deploy() { echo "QuestDB Namespace" @@ -147,19 +144,18 @@ function qdb_undeploy() { echo } -# TODO: implement method to drop table -#function qdb_drop_table() { -# echo "Drop table if exists" -# QDB_CLIENT_URL="postgresql://${QDB_USERNAME}:${QDB_PASSWORD}@questdb-0:${QDB_SQL_PORT}/defaultdb?sslmode=require" -# kubectl exec -it --namespace ${QDB_NAMESPACE} questdb-0 -- \ -# ./qdb sql --certs-dir=/qdb/qdb-certs --url=${QDB_CLIENT_URL} \ -# --execute "DROP TABLE IF EXISTS ${QDB_TABLE};" -# echo -#} +function qdb_drop_tables() { + QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}') + QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') + + echo "Drop tables, if exist" + curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;" + echo +} if [ "$QDB_REDEPLOY" == "YES" ]; then qdb_undeploy -#elif [ "$QDB_DROP_TABLE_IF_EXISTS" == "YES" ]; then -# qdb_drop_table +elif [ "$QDB_DROP_TABLES_IF_EXIST" == "YES" ]; then + qdb_drop_tables fi qdb_deploy diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 1f62adcd527fe1121a44253c02535e936212c47b..2bacc8cacb18c3cba10247472798dc0644aab2bf 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -18,18 +18,21 @@ # Read deployment settings ######################################################################################################################## + +# ----- TeraFlowSDN ------------------------------------------------------------ + # If not already set, set the URL of the Docker registry where the images will be uploaded to. # By default, assume internal MicroK8s registry is used. export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"} -# If not already set, set the list of components you want to build images for, and deploy. +# If not already set, set the list of components, separated by spaces, you want to build images for, and deploy. # By default, only basic components are deployed -export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"} +export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device automation monitoring pathcomp service slice compute webui load_generator"} # If not already set, set the tag you want to use for your images. export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"} -# If not already set, set the name of the Kubernetes namespace to deploy to. +# If not already set, set the name of the Kubernetes namespace to deploy TFS to. export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} # If not already set, set additional manifest files to be applied after the deployment @@ -38,10 +41,13 @@ export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""} # If not already set, set the new Grafana admin password export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"} -# If not already set, disable skip-build flag. +# If not already set, disable skip-build flag to rebuild the Docker images. # If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used. export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""} + +# ----- CockroachDB ------------------------------------------------------------ + # If not already set, set the namespace where CockroackDB will be deployed. export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"} @@ -54,20 +60,26 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} # If not already set, set the database name to be used by Context. export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"} + +# ----- NATS ------------------------------------------------------------------- + # If not already set, set the namespace where NATS will be deployed. export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"} + +# ----- QuestDB ---------------------------------------------------------------- + # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} -# If not already set, set the database username to be used by Monitoring. +# If not already set, set the database username to be used for QuestDB. export QDB_USERNAME=${QDB_USERNAME:-"admin"} -# If not already set, set the database user's password to be used by Monitoring. +# If not already set, set the database user's password to be used for QuestDB. export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"} +# If not already set, set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"} ######################################################################################################################## @@ -118,7 +130,7 @@ kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type=' --from-literal=METRICSDB_REST_PORT=${QDB_HTTP_PORT} \ --from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \ --from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \ - --from-literal=METRICSDB_TABLE=${QDB_TABLE} \ + --from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \ --from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \ --from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD} printf "\n" @@ -301,7 +313,8 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring" # Configure Grafana Admin Password # Ref: https://grafana.com/docs/grafana/latest/http_api/user/#change-password GRAFANA_URL_DEFAULT="http://${GRAFANA_USERNAME}:${GRAFANA_PASSWORD}@${GRAFANA_URL}" - echo "Connecting to grafana at URL: ${GRAFANA_URL_DEFAULT}..." + + echo ">> Updating Grafana 'admin' password..." curl -X PUT -H "Content-Type: application/json" -d '{ "oldPassword": "'${GRAFANA_PASSWORD}'", "newPassword": "'${TFS_GRAFANA_PASSWORD}'", @@ -314,15 +327,14 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring" echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT # Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/ - # TODO: replace user, password and database by variables to be saved QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}" - echo "Creating a datasource..." + echo ">> Creating datasources..." curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{ "access" : "proxy", "type" : "postgres", - "name" : "questdb", + "name" : "questdb-mon-kpi", "url" : "'${QDB_HOST_PORT}'", - "database" : "'${QDB_TABLE}'", + "database" : "'${QDB_TABLE_MONITORING_KPIS}'", "user" : "'${QDB_USERNAME}'", "basicAuth": false, "isDefault": true, @@ -342,16 +354,17 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring" }' ${GRAFANA_URL_UPDATED}/api/datasources echo - # Create Monitoring Dashboard + echo ">> Creating dashboards..." # Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/ - curl -X POST -H "Content-Type: application/json" \ - -d '@src/webui/grafana_dashboard_psql.json' \ + curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \ ${GRAFANA_URL_UPDATED}/api/dashboards/db echo - DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tf-l3-monit" + echo ">> Staring dashboards..." + DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit" DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id') curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID} + echo printf "\n\n" fi diff --git a/my_deploy.sh b/my_deploy.sh index 6f0e64afe311b8e56446caabfac6329024c207a9..1efea75bb3fb008e4a54d42135436a7373fd926e 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -56,7 +56,7 @@ export CRDB_DATABASE="tfs" # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" -# Disable flag for dropping database, if exists. +# Disable flag for dropping database, if it exists. export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. @@ -74,20 +74,20 @@ export NATS_REDEPLOY="" # ----- QuestDB ---------------------------------------------------------------- -# If not already set, set the namespace where QuestDB will be deployed. +# Set the namespace where QuestDB will be deployed. export QDB_NAMESPACE="qdb" -# If not already set, set the database username to be used by Monitoring. +# Set the database username to be used for QuestDB. export QDB_USERNAME="admin" -# If not already set, set the database user's password to be used by Monitoring. +# Set the database user's password to be used for QuestDB. export QDB_PASSWORD="quest" -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE="tfs_monitoring" +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" -## If not already set, disable flag for dropping table if exists. -#export QDB_DROP_TABLE_IF_EXISTS="" +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="" -# If not already set, disable flag for re-deploying QuestDB from scratch. +# Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index ff620c53425f8f447dcb00ea03bc4c9f8ce4c5e9..7c3a14975d9c7bf7d5d46be917203338bea7f1f9 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -56,7 +56,7 @@ unit_test monitoring: - docker pull questdb/questdb - docker run --name questdb -d -p 9000:9000 -p 9009:9009 -p 8812:8812 -p 9003:9003 -e QDB_CAIRO_COMMIT_LAG=1000 -e QDB_CAIRO_MAX_UNCOMMITTED_ROWS=100000 --network=teraflowbridge --rm questdb/questdb - sleep 10 - - docker run --name $IMAGE_NAME -d -p 7070:7070 --env METRICSDB_HOSTNAME=questdb --env METRICSDB_ILP_PORT=9009 --env METRICSDB_REST_PORT=9000 --env METRICSDB_TABLE=monitoring -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker run --name $IMAGE_NAME -d -p 7070:7070 --env METRICSDB_HOSTNAME=questdb --env METRICSDB_ILP_PORT=9009 --env METRICSDB_REST_PORT=9000 --env METRICSDB_TABLE_MONITORING_KPIS=tfs_monitoring_kpis -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 30 - docker ps -a - docker logs $IMAGE_NAME diff --git a/src/monitoring/service/MonitoringServiceServicerImpl.py b/src/monitoring/service/MonitoringServiceServicerImpl.py index 0bbce15094b87a17e332aad21bf34a565e8dd087..f408734df40c1bc5c16b7e108e3ce5a211165f71 100644 --- a/src/monitoring/service/MonitoringServiceServicerImpl.py +++ b/src/monitoring/service/MonitoringServiceServicerImpl.py @@ -47,7 +47,7 @@ MONITORING_INCLUDEKPI_COUNTER = Counter('monitoring_includekpi_counter', 'Monito METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT") METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT") -METRICSDB_TABLE = os.environ.get("METRICSDB_TABLE") +METRICSDB_TABLE_MONITORING_KPIS = os.environ.get("METRICSDB_TABLE_MONITORING_KPIS") class MonitoringServiceServicerImpl(MonitoringServiceServicer): def __init__(self, name_mapping : NameMapping): @@ -57,7 +57,7 @@ class MonitoringServiceServicerImpl(MonitoringServiceServicer): self.management_db = ManagementDBTools.ManagementDB('monitoring.db') self.deviceClient = DeviceClient() self.metrics_db = MetricsDBTools.MetricsDB( - METRICSDB_HOSTNAME, name_mapping, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE) + METRICSDB_HOSTNAME, name_mapping, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) self.subs_manager = SubscriptionManager(self.metrics_db) self.alarm_manager = AlarmManager(self.metrics_db) LOGGER.info('MetricsDB initialized') @@ -592,8 +592,8 @@ class MonitoringServiceServicerImpl(MonitoringServiceServicer): LOGGER.info('GetInstantKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) response.kpi_id.kpi_id.uuid = "NoID" else: - query = f"SELECT kpi_id, timestamp, kpi_value FROM {METRICSDB_TABLE} WHERE kpi_id = '{kpi_id}' " \ - f"LATEST ON timestamp PARTITION BY kpi_id" + query = f"SELECT kpi_id, timestamp, kpi_value FROM {METRICSDB_TABLE_MONITORING_KPIS} " \ + f"WHERE kpi_id = '{kpi_id}' LATEST ON timestamp PARTITION BY kpi_id" data = self.metrics_db.run_query(query) LOGGER.debug(data) if len(data) == 0: diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index 1428b0ed56dbb24a24af8fde42e4d073a48c931d..c883f9d141fc28645761641b0ccd10294b538bd2 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -75,7 +75,7 @@ os.environ[get_env_var_name(ServiceNameEnum.MONITORING, ENVVAR_SUFIX_SERVICE_POR METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME') METRICSDB_ILP_PORT = os.environ.get('METRICSDB_ILP_PORT') METRICSDB_REST_PORT = os.environ.get('METRICSDB_REST_PORT') -METRICSDB_TABLE = os.environ.get('METRICSDB_TABLE') +METRICSDB_TABLE_MONITORING_KPIS = os.environ.get('METRICSDB_TABLE_MONITORING_KPIS') LOGGER = logging.getLogger(__name__) @@ -193,7 +193,7 @@ def management_db(): def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name return monitoring_service.monitoring_servicer.metrics_db #_metrics_db = MetricsDBTools.MetricsDB( - # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE) + # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) #return _metrics_db @pytest.fixture(scope='session') diff --git a/src/tests/benchmark/policy/deploy_specs.sh b/src/tests/benchmark/policy/deploy_specs.sh index 12a45ef92a538ff48682fe45172a27d77b2800a0..7d408f003ce411566b9bf2435d89c72ff5db1459 100755 --- a/src/tests/benchmark/policy/deploy_specs.sh +++ b/src/tests/benchmark/policy/deploy_specs.sh @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -12,21 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Set the URL of your local Docker registry where the images will be uploaded to. -export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/" + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -# Supported components are: -# context device automation policy service compute monitoring webui -# interdomain slice pathcomp dlt -# dbscanserving opticalattackmitigator opticalattackdetector -# l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui" # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" -# Set the name of the Kubernetes namespace to deploy to. +# Set the name of the Kubernetes namespace to deploy TFS to. export TFS_K8S_NAMESPACE="tfs" # Set additional manifest files to be applied after the deployment @@ -35,6 +33,60 @@ export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" # Set the new Grafana admin password export TFS_GRAFANA_PASSWORD="admin123+" -# If not already set, disable skip-build flag. -# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used. -export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""} +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" diff --git a/src/tests/ecoc22/deploy_specs.sh b/src/tests/ecoc22/deploy_specs.sh index 874774e1ca50830832e842e49b6fff1114cb85d8..6c3d9db662a8232f1fcccf3835b98d69571b6337 100755 --- a/src/tests/ecoc22/deploy_specs.sh +++ b/src/tests/ecoc22/deploy_specs.sh @@ -20,7 +20,6 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -#export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui load_generator" export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui" # Set the tag you want to use for your images. @@ -57,7 +56,7 @@ export CRDB_DATABASE="tfs" # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" -# Disable flag for dropping database, if exists. +# Disable flag for dropping database, if it exists. export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. @@ -75,20 +74,20 @@ export NATS_REDEPLOY="" # ----- QuestDB ---------------------------------------------------------------- -# If not already set, set the namespace where QuestDB will be deployed. +# Set the namespace where QuestDB will be deployed. export QDB_NAMESPACE="qdb" -# If not already set, set the database username to be used by Monitoring. +# Set the database username to be used for QuestDB. export QDB_USERNAME="admin" -# If not already set, set the database user's password to be used by Monitoring. +# Set the database user's password to be used for QuestDB. export QDB_PASSWORD="quest" -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE="tfs_monitoring" +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" -## If not already set, disable flag for dropping table if exists. -#export QDB_DROP_TABLE_IF_EXISTS="" +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="" -# If not already set, disable flag for re-deploying QuestDB from scratch. +# Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" diff --git a/src/tests/ofc22/deploy_specs.sh b/src/tests/ofc22/deploy_specs.sh index 874774e1ca50830832e842e49b6fff1114cb85d8..6c3d9db662a8232f1fcccf3835b98d69571b6337 100755 --- a/src/tests/ofc22/deploy_specs.sh +++ b/src/tests/ofc22/deploy_specs.sh @@ -20,7 +20,6 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -#export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui load_generator" export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui" # Set the tag you want to use for your images. @@ -57,7 +56,7 @@ export CRDB_DATABASE="tfs" # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" -# Disable flag for dropping database, if exists. +# Disable flag for dropping database, if it exists. export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. @@ -75,20 +74,20 @@ export NATS_REDEPLOY="" # ----- QuestDB ---------------------------------------------------------------- -# If not already set, set the namespace where QuestDB will be deployed. +# Set the namespace where QuestDB will be deployed. export QDB_NAMESPACE="qdb" -# If not already set, set the database username to be used by Monitoring. +# Set the database username to be used for QuestDB. export QDB_USERNAME="admin" -# If not already set, set the database user's password to be used by Monitoring. +# Set the database user's password to be used for QuestDB. export QDB_PASSWORD="quest" -# If not already set, set the table name to be used by Monitoring. -export QDB_TABLE="tfs_monitoring" +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" -## If not already set, disable flag for dropping table if exists. -#export QDB_DROP_TABLE_IF_EXISTS="" +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="" -# If not already set, disable flag for re-deploying QuestDB from scratch. +# Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" diff --git a/src/webui/grafana_dashboard_psql.json b/src/webui/grafana_db_mon_kpis_psql.json similarity index 91% rename from src/webui/grafana_dashboard_psql.json rename to src/webui/grafana_db_mon_kpis_psql.json index ec89c1647cc1086140b0bbd35354546c405ce910..750e5254ea1e4e689d92fc39cedd22a5ee619e03 100644 --- a/src/webui/grafana_dashboard_psql.json +++ b/src/webui/grafana_db_mon_kpis_psql.json @@ -33,7 +33,7 @@ { "datasource": { "type": "postgres", - "uid": "questdb" + "uid": "questdb-mon-kpi" }, "fieldConfig": { "defaults": { @@ -162,14 +162,14 @@ { "datasource": { "type": "postgres", - "uid": "questdb" + "uid": "questdb-mon-kpi" }, "format": "time_series", "group": [], "hide": false, "metricColumn": "kpi_value", "rawQuery": true, - "rawSql": "SELECT\r\n $__time(timestamp), kpi_value AS metric, device_name, endpoint_name, kpi_sample_type\r\nFROM\r\n tfs_monitoring\r\nWHERE\r\n $__timeFilter(timestamp) AND device_name IN (${device_name}) AND endpoint_name IN (${endpoint_name}) AND kpi_sample_type IN (${kpi_sample_type})\r\nGROUP BY\r\n device_name, endpoint_name, kpi_sample_type\r\nORDER BY\r\n timestamp", + "rawSql": "SELECT\r\n $__time(timestamp), kpi_value AS metric, device_name, endpoint_name, kpi_sample_type\r\nFROM\r\n tfs_monitoring_kpis\r\nWHERE\r\n $__timeFilter(timestamp) AND device_name IN (${device_name}) AND endpoint_name IN (${endpoint_name}) AND kpi_sample_type IN (${kpi_sample_type})\r\nGROUP BY\r\n device_name, endpoint_name, kpi_sample_type\r\nORDER BY\r\n timestamp", "refId": "A", "select": [ [ @@ -181,7 +181,7 @@ } ] ], - "table": "monitoring", + "table": "tfs_monitoring_kpis", "timeColumn": "timestamp", "where": [ { @@ -227,16 +227,16 @@ }, "datasource": { "type": "postgres", - "uid": "questdb" + "uid": "questdb-mon-kpi" }, - "definition": "SELECT DISTINCT device_name FROM tfs_monitoring;", + "definition": "SELECT DISTINCT device_name FROM tfs_monitoring_kpis;", "hide": 0, "includeAll": true, "label": "Device", "multi": true, "name": "device_name", "options": [], - "query": "SELECT DISTINCT device_name FROM tfs_monitoring;", + "query": "SELECT DISTINCT device_name FROM tfs_monitoring_kpis;", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -255,16 +255,16 @@ }, "datasource": { "type": "postgres", - "uid": "questdb" + "uid": "questdb-mon-kpi" }, - "definition": "SELECT DISTINCT endpoint_name FROM tfs_monitoring WHERE device_name IN (${device_name})", + "definition": "SELECT DISTINCT endpoint_name FROM tfs_monitoring_kpis WHERE device_name IN (${device_name})", "hide": 0, "includeAll": true, "label": "EndPoint", "multi": true, "name": "endpoint_name", "options": [], - "query": "SELECT DISTINCT endpoint_name FROM tfs_monitoring WHERE device_name IN (${device_name})", + "query": "SELECT DISTINCT endpoint_name FROM tfs_monitoring_kpis WHERE device_name IN (${device_name})", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -283,16 +283,16 @@ }, "datasource": { "type": "postgres", - "uid": "questdb" + "uid": "questdb-mon-kpi" }, - "definition": "SELECT DISTINCT kpi_sample_type FROM tfs_monitoring;", + "definition": "SELECT DISTINCT kpi_sample_type FROM tfs_monitoring_kpis;", "hide": 0, "includeAll": true, "label": "Kpi Sample Type", "multi": true, "name": "kpi_sample_type", "options": [], - "query": "SELECT DISTINCT kpi_sample_type FROM tfs_monitoring;", + "query": "SELECT DISTINCT kpi_sample_type FROM tfs_monitoring_kpis;", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -308,7 +308,7 @@ "timepicker": {}, "timezone": "utc", "title": "L3 Monitoring", - "uid": "tf-l3-monit", + "uid": "tfs-l3-monit", "version": 6, "weekStart": "" } diff --git a/src/webui/grafana_backup_dashboard.json b/src/webui/old/grafana_backup_dashboard.json similarity index 100% rename from src/webui/grafana_backup_dashboard.json rename to src/webui/old/grafana_backup_dashboard.json diff --git a/src/webui/grafana_dashboard.json b/src/webui/old/grafana_dashboard.json similarity index 100% rename from src/webui/grafana_dashboard.json rename to src/webui/old/grafana_dashboard.json