Merge branch 'develop' of https://labs.etsi.org/rep/tfs/controller into... (b69b618a) · Commits · TFS / controller

.gitignore

+4 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@ share/python-wheels/
		.installed.cfg
		*.egg
		MANIFEST
		.my_venv/
		# requirements.txt # removed to enable tracking versions of packages over time

		# PyInstaller
		@@ -176,3 +177,6 @@ libyang/

		# Other logs
		*/logs/.log.*

		# PySpark checkpoints
		src/analytics/.spark/*

.gitlab-ci.yml

+3 −0

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ include:
		- local: '/src/opticalcontroller/.gitlab-ci.yml'
		- local: '/src/ztp/.gitlab-ci.yml'
		- local: '/src/policy/.gitlab-ci.yml'
		- local: '/src/automation/.gitlab-ci.yml'
		- local: '/src/forecaster/.gitlab-ci.yml'
		#- local: '/src/webui/.gitlab-ci.yml'
		#- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'
		@@ -48,6 +49,8 @@ include:
		- local: '/src/kpi_manager/.gitlab-ci.yml'
		- local: '/src/kpi_value_api/.gitlab-ci.yml'
		- local: '/src/kpi_value_writer/.gitlab-ci.yml'
		- local: '/src/telemetry/.gitlab-ci.yml'
		- local: '/src/analytics/.gitlab-ci.yml'
		- local: '/src/qos_profile/.gitlab-ci.yml'

		# This should be last one: end-to-end integration tests

deploy/all.sh

+4 −1

Original line number	Diff line number	Diff line
		@@ -33,7 +33,7 @@ export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice n
		#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"

		# Uncomment to activate Monitoring Framework (new)
		#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api"
		#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics"

		# Uncomment to activate BGP-LS Speaker
		#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker"
		@@ -215,6 +215,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
		# Deploy QuestDB
		./deploy/qdb.sh

		# Deploy Apache Kafka
		./deploy/kafka.sh

		# Expose Dashboard
		./deploy/expose_dashboard.sh

deploy/crdb.sh

+34 −12

Original line number	Diff line number	Diff line
		@@ -154,8 +154,8 @@ function crdb_undeploy_single() {
		echo
		}

		function crdb_drop_database_single() {
		echo "Drop database if exists"
		function crdb_drop_databases_single() {
		echo "Drop TFS databases, if exist"

		if [[ -z "${GITLAB_CI}" ]]; then
		#kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml
		@@ -168,9 +168,20 @@ function crdb_drop_database_single() {

		CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST}:${CRDB_PORT}/defaultdb?sslmode=require"
		echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}"

		DATABASES=$(
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
		--execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
		--execute "SHOW DATABASES;" --format=tsv \| awk '{print $1}' \| grep "^tfs"
		)
		echo "Found TFS databases: ${DATABASES}" \| tr '\n' ' '

		for DB_NAME in $DATABASES; do
		echo "Dropping TFS database: $DB_NAME"
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
		--execute="DROP DATABASE IF EXISTS $DB_NAME CASCADE;"
		done
		echo
		}

		@@ -349,11 +360,22 @@ function crdb_undeploy_cluster() {
		echo
		}

		function crdb_drop_database_cluster() {
		echo "Drop database if exists"
		function crdb_drop_databases_cluster() {
		echo "Drop TFS databases, if exist"

		DATABASES=$(
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public --execute \
		"DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public \
		--execute "SHOW DATABASES;" --format=tsv \| awk '{print $1}' \| grep "^tfs"
		)
		echo "Found TFS databases: ${DATABASES}" \| tr '\n' ' '

		for DB_NAME in $DATABASES; do
		echo "Dropping TFS database: $DB_NAME"
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public \
		--execute="DROP DATABASE IF EXISTS $DB_NAME CASCADE;"
		done
		echo
		}

		@@ -365,7 +387,7 @@ if [ "$CRDB_DEPLOY_MODE" == "single" ]; then
		crdb_deploy_single

		if [ "$CRDB_DROP_DATABASE_IF_EXISTS" == "YES" ]; then
		crdb_drop_database_single
		crdb_drop_databases_single
		fi
		elif [ "$CRDB_DEPLOY_MODE" == "cluster" ]; then
		if [ "$CRDB_REDEPLOY" == "YES" ]; then
		@@ -375,7 +397,7 @@ elif [ "$CRDB_DEPLOY_MODE" == "cluster" ]; then
		crdb_deploy_cluster

		if [ "$CRDB_DROP_DATABASE_IF_EXISTS" == "YES" ]; then
		crdb_drop_database_cluster
		crdb_drop_databases_cluster
		fi
		else
		echo "Unsupported value: CRDB_DEPLOY_MODE=$CRDB_DEPLOY_MODE"

deploy/kafka.sh

+53 −32

Original line number	Diff line number	Diff line
		@@ -20,6 +20,12 @@
		# If not already set, set the namespace where Apache Kafka will be deployed.
		export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"}

		# If not already set, set the port Apache Kafka server will be exposed to.
		export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"}

		# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost.
		export KFK_REDEPLOY=${KFK_REDEPLOY:-""}


		########################################################################################################################
		# Automated steps start here
		@@ -35,18 +41,19 @@ KFK_MANIFEST="02-kafka.yaml"
		TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests"
		mkdir -p ${TMP_MANIFESTS_FOLDER}

		function kafka_deploy() {
		# copy zookeeper and kafka manifest files to temporary manifest location
		cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
		cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"

		echo "Apache Kafka Namespace"
		echo ">>> Delete Apache Kafka Namespace"
		# echo "Apache Kafka Namespace"
		echo "Delete Apache Kafka Namespace"
		kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found

		echo ">>> Create Apache Kafka Namespace"
		echo "Create Apache Kafka Namespace"
		kubectl create namespace ${KFK_NAMESPACE}

		echo ">>> Deplying Apache Kafka Zookeeper"
		# echo ">>> Deplying Apache Kafka Zookeeper"
		# Kafka zookeeper service should be deployed before the kafka service
		kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"

		@@ -56,14 +63,28 @@ KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEE
		# Kafka service should be deployed after the zookeeper service
		sed -i "s/<ZOOKEEPER_INTERNAL_IP>/${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"

		echo ">>> Deploying Apache Kafka Broker"
		# echo ">>> Deploying Apache Kafka Broker"
		kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"

		echo ">>> Verifing Apache Kafka deployment"
		sleep 10
		KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods)
		if echo "$KFK_PODS_STATUS" \| grep -qEv 'STATUS\|Running'; then
		echo "Deployment Error: \n $KFK_PODS_STATUS"
		# echo ">>> Verifing Apache Kafka deployment"
		sleep 5
		# KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods)
		# if echo "$KFK_PODS_STATUS" \| grep -qEv 'STATUS\|Running'; then
		# echo "Deployment Error: \n $KFK_PODS_STATUS"
		# else
		# echo "$KFK_PODS_STATUS"
		# fi
		}

		echo ">>> Apache Kafka"
		echo "Checking if Apache Kafka is deployed ... "
		if [ "$KFK_REDEPLOY" == "YES" ]; then
		echo "Redeploying kafka namespace"
		kafka_deploy
		elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then
		echo "Apache Kafka already present; skipping step."
		else
		echo "$KFK_PODS_STATUS"
		echo "Kafka namespace doesn't exists. Deploying kafka namespace"
		kafka_deploy
		fi
		echo