Merge branch 'develop' of ssh://gifrerenom_labs.etsi.org/tfs/controller into... (4c9d57e8) · Commits · TFS / controller

.gitignore

+1 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@ share/python-wheels/
		.installed.cfg
		*.egg
		MANIFEST
		.my_venv/
		# requirements.txt # removed to enable tracking versions of packages over time

		# PyInstaller

.gitlab-ci.yml

+4 −0

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ include:
		- local: '/src/opticalcontroller/.gitlab-ci.yml'
		- local: '/src/ztp/.gitlab-ci.yml'
		- local: '/src/policy/.gitlab-ci.yml'
		- local: '/src/automation/.gitlab-ci.yml'
		- local: '/src/forecaster/.gitlab-ci.yml'
		#- local: '/src/webui/.gitlab-ci.yml'
		#- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'
		@@ -49,5 +50,8 @@ include:
		- local: '/src/kpi_value_api/.gitlab-ci.yml'
		- local: '/src/kpi_value_writer/.gitlab-ci.yml'
		- local: '/src/telemetry/.gitlab-ci.yml'
		- local: '/src/analytics/.gitlab-ci.yml'
		- local: '/src/qos_profile/.gitlab-ci.yml'

		# This should be last one: end-to-end integration tests
		- local: '/src/tests/.gitlab-ci.yml'

deploy/crdb.sh

+34 −12

Original line number	Diff line number	Diff line
		@@ -154,8 +154,8 @@ function crdb_undeploy_single() {
		echo
		}

		function crdb_drop_database_single() {
		echo "Drop database if exists"
		function crdb_drop_databases_single() {
		echo "Drop TFS databases, if exist"

		if [[ -z "${GITLAB_CI}" ]]; then
		#kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml
		@@ -168,9 +168,20 @@ function crdb_drop_database_single() {

		CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST}:${CRDB_PORT}/defaultdb?sslmode=require"
		echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}"

		DATABASES=$(
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
		--execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
		--execute "SHOW DATABASES;" --format=tsv \| awk '{print $1}' \| grep "^tfs"
		)
		echo "Found TFS databases: ${DATABASES}" \| tr '\n' ' '

		for DB_NAME in $DATABASES; do
		echo "Dropping TFS database: $DB_NAME"
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
		--execute="DROP DATABASE IF EXISTS $DB_NAME CASCADE;"
		done
		echo
		}

		@@ -349,11 +360,22 @@ function crdb_undeploy_cluster() {
		echo
		}

		function crdb_drop_database_cluster() {
		echo "Drop database if exists"
		function crdb_drop_databases_cluster() {
		echo "Drop TFS databases, if exist"

		DATABASES=$(
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public --execute \
		"DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public \
		--execute "SHOW DATABASES;" --format=tsv \| awk '{print $1}' \| grep "^tfs"
		)
		echo "Found TFS databases: ${DATABASES}" \| tr '\n' ' '

		for DB_NAME in $DATABASES; do
		echo "Dropping TFS database: $DB_NAME"
		kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \
		./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public \
		--execute="DROP DATABASE IF EXISTS $DB_NAME CASCADE;"
		done
		echo
		}

		@@ -365,7 +387,7 @@ if [ "$CRDB_DEPLOY_MODE" == "single" ]; then
		crdb_deploy_single

		if [ "$CRDB_DROP_DATABASE_IF_EXISTS" == "YES" ]; then
		crdb_drop_database_single
		crdb_drop_databases_single
		fi
		elif [ "$CRDB_DEPLOY_MODE" == "cluster" ]; then
		if [ "$CRDB_REDEPLOY" == "YES" ]; then
		@@ -375,7 +397,7 @@ elif [ "$CRDB_DEPLOY_MODE" == "cluster" ]; then
		crdb_deploy_cluster

		if [ "$CRDB_DROP_DATABASE_IF_EXISTS" == "YES" ]; then
		crdb_drop_database_cluster
		crdb_drop_databases_cluster
		fi
		else
		echo "Unsupported value: CRDB_DEPLOY_MODE=$CRDB_DEPLOY_MODE"

deploy/kafka.sh

+7 −7

Original line number	Diff line number	Diff line
		@@ -47,10 +47,10 @@ function kafka_deploy() {
		cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"

		# echo "Apache Kafka Namespace"
		echo ">>> Delete Apache Kafka Namespace"
		echo "Delete Apache Kafka Namespace"
		kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found

		echo ">>> Create Apache Kafka Namespace"
		echo "Create Apache Kafka Namespace"
		kubectl create namespace ${KFK_NAMESPACE}

		# echo ">>> Deplying Apache Kafka Zookeeper"
		@@ -76,15 +76,15 @@ function kafka_deploy() {
		# fi
		}

		echo "Apache Kafka"
		echo ">>> Checking if Apache Kafka is deployed ... "
		echo ">>> Apache Kafka"
		echo "Checking if Apache Kafka is deployed ... "
		if [ "$KFK_REDEPLOY" == "YES" ]; then
		echo ">>> Redeploying kafka namespace"
		echo "Redeploying kafka namespace"
		kafka_deploy
		elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then
		echo ">>> Apache Kafka already present; skipping step."
		echo "Apache Kafka already present; skipping step."
		else
		echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace"
		echo "Kafka namespace doesn't exists. Deploying kafka namespace"
		kafka_deploy
		fi
		echo

deploy/tfs.sh

+21 −40

Original line number	Diff line number	Diff line
		@@ -146,55 +146,17 @@ kubectl create namespace $TFS_K8S_NAMESPACE
		sleep 2
		printf "\n"

		echo "Create secret with CockroachDB data"
		echo ">>> Create Secret with CockroachDB data..."
		CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
		CRDB_DATABASE_CONTEXT=${CRDB_DATABASE} # TODO: change by specific configurable environment variable
		kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
		--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
		--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
		--from-literal=CRDB_DATABASE=${CRDB_DATABASE_CONTEXT} \
		--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
		--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
		--from-literal=CRDB_SSLMODE=require
		printf "\n"

		echo "Create secret with CockroachDB data for KPI Management microservices"
		CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
		CRDB_DATABASE_KPI_MGMT="tfs_kpi_mgmt" # TODO: change by specific configurable environment variable
		kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
		--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
		--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
		--from-literal=CRDB_DATABASE=${CRDB_DATABASE_KPI_MGMT} \
		--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
		--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
		--from-literal=CRDB_SSLMODE=require
		printf "\n"

		echo "Create secret with CockroachDB data for Telemetry microservices"
		CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
		CRDB_DATABASE_TELEMETRY="tfs_telemetry" # TODO: change by specific configurable environment variable
		kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
		--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
		--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
		--from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \
		--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
		--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
		--from-literal=CRDB_SSLMODE=require
		printf "\n"

		echo "Create secret with CockroachDB data for Analytics microservices"
		CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
		CRDB_DATABASE_ANALYTICS="tfs_analytics" # TODO: change by specific configurable environment variable
		kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
		--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
		--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
		--from-literal=CRDB_DATABASE=${CRDB_DATABASE_ANALYTICS} \
		--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
		--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
		--from-literal=CRDB_SSLMODE=require
		printf "\n"

		echo "Create secret with Apache Kafka data for KPI, Telemetry and Analytics microservices"
		echo ">>> Create Secret with Apache Kakfa..."
		KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')
		kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
		--from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
		@@ -229,6 +191,21 @@ kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type='
		--from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}
		printf "\n"

		# Check if "dlt" is in the list of components
		if [[ " ${TFS_COMPONENTS[@]} " =~ " dlt " ]]; then
		echo "Create secret for HLF keystore"
		kubectl create secret generic dlt-keystone --namespace ${TFS_K8S_NAMESPACE} --from-file=keystore=${KEY_DIRECTORY_PATH}
		printf "\n"

		echo "Create secret for HLF signcerts"
		kubectl create secret generic dlt-signcerts --namespace ${TFS_K8S_NAMESPACE} --from-file=signcerts.pem=${CERT_DIRECTORY_PATH}
		printf "\n"

		echo "Create secret for HLF ca.crt"
		kubectl create secret generic dlt-ca-crt --namespace ${TFS_K8S_NAMESPACE} --from-file=ca.crt=${TLS_CERT_PATH}
		printf "\n"
		fi

		echo "Deploying components and collecting environment variables..."
		ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh
		echo "# Environment variables for TeraFlowSDN deployment" > $ENV_VARS_SCRIPT
		@@ -654,6 +631,10 @@ if [[ "$TFS_COMPONENTS" == "monitoring" ]] && [[ "$TFS_COMPONENTS" == *"webui"
		printf "\n\n"
		fi

		echo "Pruning Docker Images..."
		docker image prune --force
		printf "\n\n"

		if [ "$DOCKER_BUILD" == "docker buildx build" ]; then
		echo "Pruning Docker Buildx Cache..."
		docker buildx prune --force