Merge branch 'develop' into feat/compute-component (510e4d68) · Commits · TFS / controller

deploy/all.sh

+18 −26

deploy/crdb.sh

+0 −3

Original line number	Original line	Diff line number	Diff line
	@@ -66,9 +66,6 @@ CRDB_MANIFESTS_PATH="manifests/cockroachdb"
	# Create a tmp folder for files modified during the deployment		# Create a tmp folder for files modified during the deployment
	TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"		TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"
	mkdir -p $TMP_MANIFESTS_FOLDER		mkdir -p $TMP_MANIFESTS_FOLDER
	TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
	mkdir -p $TMP_LOGS_FOLDER
	CRDB_LOG_FILE="$TMP_LOGS_FOLDER/crdb_deploy.log"

	function crdb_deploy_single() {		function crdb_deploy_single() {
	echo "CockroachDB Namespace"		echo "CockroachDB Namespace"

deploy/nats.sh

+0 −8

Original line number	Original line	Diff line number	Diff line
	@@ -31,14 +31,6 @@ export NATS_REDEPLOY=${NATS_REDEPLOY:-""}
	# Automated steps start here		# Automated steps start here
	########################################################################################################################		########################################################################################################################

	# Constants
	TMP_FOLDER="./tmp"
	NATS_MANIFESTS_PATH="manifests/nats"

	# Create a tmp folder for files modified during the deployment
	TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"
	mkdir -p $TMP_MANIFESTS_FOLDER

	function nats_deploy_single() {		function nats_deploy_single() {
	echo "NATS Namespace"		echo "NATS Namespace"
	echo ">>> Create NATS Namespace (if missing)"		echo ">>> Create NATS Namespace (if missing)"

deploy/qdb.sh

+25 −23

Original line number	Original line	Diff line number	Diff line
	@@ -21,20 +21,24 @@
	# If not already set, set the namespace where QuestDB will be deployed.		# If not already set, set the namespace where QuestDB will be deployed.
	export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}		export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}

	# If not already set, set the database username to be used by Monitoring.		# If not already set, set the database username to be used for QuestDB.
	export QDB_USERNAME=${QDB_USERNAME:-"admin"}		export QDB_USERNAME=${QDB_USERNAME:-"admin"}

	# If not already set, set the database user's password to be used by Monitoring.		# If not already set, set the database user's password to be used for QuestDB.
	export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}		export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}

	# If not already set, set the table name to be used by Monitoring.		# If not already set, set the table name to be used by Monitoring for KPIs.
	export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"}		export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

	## If not already set, disable flag for dropping table if exists.		# If not already set, set the table name to be used by Slice for plotting groups.
	## WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!		export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}
	## If QDB_DROP_TABLE_IF_EXISTS is "YES", the table pointed by variable QDB_TABLE will be dropped while
	## checking/deploying QuestDB.		# If not already set, disable flag for dropping tables if they exist.
	#export QDB_DROP_TABLE_IF_EXISTS=${QDB_DROP_TABLE_IF_EXISTS:-""}		# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!
			# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variables
			# QDB_TABLE_MONITORING_KPIS and QDB_TABLE_SLICE_GROUPS will be dropped
			# while checking/deploying QuestDB.
			export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}

	# If not already set, disable flag for re-deploying QuestDB from scratch.		# If not already set, disable flag for re-deploying QuestDB from scratch.
	# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!		# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!
	@@ -52,9 +56,6 @@ QDB_MANIFESTS_PATH="manifests/questdb"

	# Create a tmp folder for files modified during the deployment		# Create a tmp folder for files modified during the deployment
	TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"		TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"
	TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
	QDB_LOG_FILE="$TMP_LOGS_FOLDER/qdb_deploy.log"
	mkdir -p $TMP_LOGS_FOLDER

	function qdb_deploy() {		function qdb_deploy() {
	echo "QuestDB Namespace"		echo "QuestDB Namespace"
	@@ -147,19 +148,20 @@ function qdb_undeploy() {
	echo		echo
	}		}

	# TODO: implement method to drop table		function qdb_drop_tables() {
	#function qdb_drop_table() {		QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}')
	# echo "Drop table if exists"		QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}')
	# QDB_CLIENT_URL="postgresql://${QDB_USERNAME}:${QDB_PASSWORD}@questdb-0:${QDB_SQL_PORT}/defaultdb?sslmode=require"
	# kubectl exec -it --namespace ${QDB_NAMESPACE} questdb-0 -- \		echo "Drop tables, if exist"
	# ./qdb sql --certs-dir=/qdb/qdb-certs --url=${QDB_CLIENT_URL} \		curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;"
	# --execute "DROP TABLE IF EXISTS ${QDB_TABLE};"		echo
	# echo		curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;"
	#}		echo
			}

	if [ "$QDB_REDEPLOY" == "YES" ]; then		if [ "$QDB_REDEPLOY" == "YES" ]; then
	qdb_undeploy		qdb_undeploy
	#elif [ "$QDB_DROP_TABLE_IF_EXISTS" == "YES" ]; then		elif [ "$QDB_DROP_TABLES_IF_EXIST" == "YES" ]; then
	# qdb_drop_table		qdb_drop_tables
	fi		fi
	qdb_deploy		qdb_deploy

deploy/tfs.sh

+76 −19

Original line number	Original line	Diff line number	Diff line
	@@ -18,18 +18,21 @@
	# Read deployment settings		# Read deployment settings
	########################################################################################################################		########################################################################################################################


			# ----- TeraFlowSDN ------------------------------------------------------------

	# If not already set, set the URL of the Docker registry where the images will be uploaded to.		# If not already set, set the URL of the Docker registry where the images will be uploaded to.
	# By default, assume internal MicroK8s registry is used.		# By default, assume internal MicroK8s registry is used.
	export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}		export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}

	# If not already set, set the list of components you want to build images for, and deploy.		# If not already set, set the list of components, separated by spaces, you want to build images for, and deploy.
	# By default, only basic components are deployed		# By default, only basic components are deployed
	export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"}		export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device automation monitoring pathcomp service slice compute webui load_generator"}

	# If not already set, set the tag you want to use for your images.		# If not already set, set the tag you want to use for your images.
	export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}		export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}

	# If not already set, set the name of the Kubernetes namespace to deploy to.		# If not already set, set the name of the Kubernetes namespace to deploy TFS to.
	export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}		export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}

	# If not already set, set additional manifest files to be applied after the deployment		# If not already set, set additional manifest files to be applied after the deployment
	@@ -38,10 +41,13 @@ export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}
	# If not already set, set the new Grafana admin password		# If not already set, set the new Grafana admin password
	export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}		export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}

	# If not already set, disable skip-build flag.		# If not already set, disable skip-build flag to rebuild the Docker images.
	# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.		# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
	export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}		export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}


			# ----- CockroachDB ------------------------------------------------------------

	# If not already set, set the namespace where CockroackDB will be deployed.		# If not already set, set the namespace where CockroackDB will be deployed.
	export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}		export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}

	@@ -54,20 +60,29 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"}
	# If not already set, set the database name to be used by Context.		# If not already set, set the database name to be used by Context.
	export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}		export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}


			# ----- NATS -------------------------------------------------------------------

	# If not already set, set the namespace where NATS will be deployed.		# If not already set, set the namespace where NATS will be deployed.
	export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}		export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}


			# ----- QuestDB ----------------------------------------------------------------

	# If not already set, set the namespace where QuestDB will be deployed.		# If not already set, set the namespace where QuestDB will be deployed.
	export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}		export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}

	# If not already set, set the database username to be used by Monitoring.		# If not already set, set the database username to be used for QuestDB.
	export QDB_USERNAME=${QDB_USERNAME:-"admin"}		export QDB_USERNAME=${QDB_USERNAME:-"admin"}

	# If not already set, set the database user's password to be used by Monitoring.		# If not already set, set the database user's password to be used for QuestDB.
	export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}		export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}

	# If not already set, set the table name to be used by Monitoring.		# If not already set, set the table name to be used by Monitoring for KPIs.
	export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"}		export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

			# If not already set, set the table name to be used by Slice for plotting groups.
			export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}


	########################################################################################################################		########################################################################################################################
	@@ -85,7 +100,7 @@ TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
	mkdir -p $TMP_LOGS_FOLDER		mkdir -p $TMP_LOGS_FOLDER

	echo "Deleting and Creating a new namespace..."		echo "Deleting and Creating a new namespace..."
	kubectl delete namespace $TFS_K8S_NAMESPACE		kubectl delete namespace $TFS_K8S_NAMESPACE --ignore-not-found
	kubectl create namespace $TFS_K8S_NAMESPACE		kubectl create namespace $TFS_K8S_NAMESPACE
	printf "\n"		printf "\n"

	@@ -118,7 +133,8 @@ kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type='
	--from-literal=METRICSDB_REST_PORT=${QDB_HTTP_PORT} \		--from-literal=METRICSDB_REST_PORT=${QDB_HTTP_PORT} \
	--from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \		--from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \
	--from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \		--from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \
	--from-literal=METRICSDB_TABLE=${QDB_TABLE} \		--from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \
			--from-literal=METRICSDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS} \
	--from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \		--from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \
	--from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}		--from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}
	printf "\n"		printf "\n"
	@@ -301,28 +317,34 @@ if [[ "$TFS_COMPONENTS" == "webui" ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
	# Configure Grafana Admin Password		# Configure Grafana Admin Password
	# Ref: https://grafana.com/docs/grafana/latest/http_api/user/#change-password		# Ref: https://grafana.com/docs/grafana/latest/http_api/user/#change-password
	GRAFANA_URL_DEFAULT="http://${GRAFANA_USERNAME}:${GRAFANA_PASSWORD}@${GRAFANA_URL}"		GRAFANA_URL_DEFAULT="http://${GRAFANA_USERNAME}:${GRAFANA_PASSWORD}@${GRAFANA_URL}"
	echo "Connecting to grafana at URL: ${GRAFANA_URL_DEFAULT}..."
			echo ">> Updating Grafana 'admin' password..."
	curl -X PUT -H "Content-Type: application/json" -d '{		curl -X PUT -H "Content-Type: application/json" -d '{
	"oldPassword": "'${GRAFANA_PASSWORD}'",		"oldPassword": "'${GRAFANA_PASSWORD}'",
	"newPassword": "'${TFS_GRAFANA_PASSWORD}'",		"newPassword": "'${TFS_GRAFANA_PASSWORD}'",
	"confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"		"confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"
	}' ${GRAFANA_URL_DEFAULT}/api/user/password		}' ${GRAFANA_URL_DEFAULT}/api/user/password
	echo		echo
			echo

	# Updated Grafana API URL		# Updated Grafana API URL
	GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"		GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"
	echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT		echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT

			echo ">> Installing Scatter Plot plugin..."
			curl -X POST -H "Content-Type: application/json" -H "Content-Length: 0" \
			${GRAFANA_URL_UPDATED}/api/plugins/michaeldmoore-scatter-panel/install
			echo

	# Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/		# Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/
	# TODO: replace user, password and database by variables to be saved
	QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"		QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"
	echo "Creating a datasource..."		echo ">> Creating datasources..."
	curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{		curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
	"access" : "proxy",		"access" : "proxy",
	"type" : "postgres",		"type" : "postgres",
	"name" : "questdb",		"name" : "questdb-mon-kpi",
	"url" : "'${QDB_HOST_PORT}'",		"url" : "'${QDB_HOST_PORT}'",
	"database" : "'${QDB_TABLE}'",		"database" : "'${QDB_TABLE_MONITORING_KPIS}'",
	"user" : "'${QDB_USERNAME}'",		"user" : "'${QDB_USERNAME}'",
	"basicAuth": false,		"basicAuth": false,
	"isDefault": true,		"isDefault": true,
	@@ -342,16 +364,51 @@ if [[ "$TFS_COMPONENTS" == "webui" ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
	}' ${GRAFANA_URL_UPDATED}/api/datasources		}' ${GRAFANA_URL_UPDATED}/api/datasources
	echo		echo

	# Create Monitoring Dashboard		curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
			"access" : "proxy",
			"type" : "postgres",
			"name" : "questdb-slc-grp",
			"url" : "'${QDB_HOST_PORT}'",
			"database" : "'${QDB_TABLE_SLICE_GROUPS}'",
			"user" : "'${QDB_USERNAME}'",
			"basicAuth": false,
			"isDefault": false,
			"jsonData" : {
			"sslmode" : "disable",
			"postgresVersion" : 1100,
			"maxOpenConns" : 0,
			"maxIdleConns" : 2,
			"connMaxLifetime" : 14400,
			"tlsAuth" : false,
			"tlsAuthWithCACert" : false,
			"timescaledb" : false,
			"tlsConfigurationMethod": "file-path",
			"tlsSkipVerify" : true
			},
			"secureJsonData": {"password": "'${QDB_PASSWORD}'"}
			}' ${GRAFANA_URL_UPDATED}/api/datasources
			printf "\n\n"

			echo ">> Creating dashboards..."
	# Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/		# Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/
	curl -X POST -H "Content-Type: application/json" \		curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \
	-d '@src/webui/grafana_dashboard_psql.json' \		${GRAFANA_URL_UPDATED}/api/dashboards/db
			echo

			curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_slc_grps_psql.json' \
	${GRAFANA_URL_UPDATED}/api/dashboards/db		${GRAFANA_URL_UPDATED}/api/dashboards/db
			printf "\n\n"

			echo ">> Staring dashboards..."
			DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit"
			DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')
			curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
	echo		echo

	DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tf-l3-monit"		DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-slice-grps"
	DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')		DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')
	curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
			echo

	printf "\n\n"		printf "\n\n"
	fi		fi