Release 2.0.0

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

[run]

data_file = ~/teraflow/controller/coverage/.coverage

data_file = ~/tfs-ctrl/coverage/.coverage

source = .

omit =

    */proto/*

    raise\ NotImplementedError

[html]

directory = ~/teraflow/controller/coverage/html_report

directory = ~/tfs-ctrl/coverage/html_report

[xml]

output = ~/teraflow/controller/coverage/report.xml

!.gitkeep

*.dot

*.xml

#!/bin/bash

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

########################################################################################################################

# Read deployment settings

########################################################################################################################

# ----- TeraFlowSDN ------------------------------------------------------------

# If not already set, set the URL of the Docker registry where the images will be uploaded to.

# By default, assume internal MicroK8s registry is used.

export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}

# If not already set, set the list of components you want to build images for, and deploy.

# By default, only basic components are deployed

export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"}

# If not already set, set the tag you want to use for your images.

export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}

# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}

# If not already set, set additional manifest files to be applied after the deployment

export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}

# If not already set, set the new Grafana admin password

export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}

# If not already set, disable skip-build flag.

# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.

export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}

# ----- CockroachDB ------------------------------------------------------------

# If not already set, set the namespace where CockroackDB will be deployed.

export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}

# If not already set, set the database username to be used by Context.

export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"}

# If not already set, set the database user's password to be used by Context.

export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"}

# If not already set, set the database name to be used by Context.

export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}

# If not already set, set the name of the secret where CockroachDB data and credentials will be stored.

export CRDB_SECRET_NAME=${CRDB_SECRET_NAME:-"crdb-data"}

# If not already set, set the namespace where the secret containing CockroachDB data and credentials will be stored.

export CRDB_SECRET_NAMESPACE=${CRDB_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}}

# If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'.

# "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

# - If CRDB_DEPLOY_MODE is "single", CockroachDB is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If CRDB_DEPLOY_MODE is "cluster", CockroachDB is deployed in cluster mode, and an entire CockroachDB cluster

#   with 3 replicas and version v22.2 (set by default) will be deployed. It is convenient for production and

#   provides scalability features. If you are deploying for production, also read the following link providing

#   details on deploying CockroachDB for production environments:

#   Ref: https://www.cockroachlabs.com/docs/stable/recommended-production-settings.html

export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"}

# If not already set, disable flag for dropping database if exists.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

export CRDB_DROP_DATABASE_IF_EXISTS=${CRDB_DROP_DATABASE_IF_EXISTS:-""}

# If not already set, disable flag for re-deploying CockroachDB from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# WARNING: THE REDEPLOY MIGHT TAKE FEW MINUTES TO COMPLETE GRACEFULLY IN CLUSTER MODE

# If CRDB_REDEPLOY is "YES", the database will be dropped while checking/deploying CockroachDB.

export CRDB_REDEPLOY=${CRDB_REDEPLOY:-""}

# ----- NATS -------------------------------------------------------------------

# If not already set, set the namespace where NATS will be deployed.

export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}

# If not already set, set the name of the secret where NATS data and credentials will be stored.

export NATS_SECRET_NAME=${NATS_SECRET_NAME:-"nats-data"}

# If not already set, set the namespace where the secret containing NATS data and credentials will be stored.

export NATS_SECRET_NAMESPACE=${NATS_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}}

# If not already set, disable flag for re-deploying NATS from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION!

# If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS.

export NATS_REDEPLOY=${NATS_REDEPLOY:-""}

# ----- QuestDB ----------------------------------------------------------------

# If not already set, set the namespace where QuestDB will be deployed.

export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}

# If not already set, set the database username to be used by Monitoring.

export QDB_USERNAME=${QDB_USERNAME:-"admin"}

# If not already set, set the database user's password to be used by Monitoring.

export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}

# If not already set, set the table name to be used by Monitoring.

export QDB_TABLE=${QDB_TABLE:-"tfs_monitoring"}

## If not already set, disable flag for dropping table if exists.

## WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!

## If QDB_DROP_TABLE_IF_EXISTS is "YES", the table pointed by variable QDB_TABLE will be dropped while

## checking/deploying QuestDB.

#export QDB_DROP_TABLE_IF_EXISTS=${QDB_DROP_TABLE_IF_EXISTS:-""}

# If not already set, disable flag for re-deploying QuestDB from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If QDB_REDEPLOY is "YES", the database will be dropped while checking/deploying QuestDB.

export QDB_REDEPLOY=${QDB_REDEPLOY:-""}

########################################################################################################################

# Automated steps start here

########################################################################################################################

# Deploy CockroachDB

./deploy/crdb.sh

# Deploy NATS

./deploy/nats.sh

# Deploy QuestDB

./deploy/qdb.sh

# Deploy TeraFlowSDN

./deploy/tfs.sh

# Show deploy summary

./deploy/show.sh

echo "Done!"

#!/bin/bash

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

########################################################################################################################

# Read deployment settings

########################################################################################################################

# If not already set, set the URL of the Docker registry where the images will be uploaded to.

# By default, assume internal MicroK8s registry is used.

export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}

TFS_COMPONENTS=$1

# If not already set, set the tag you want to use for your images.

export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}

# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}

# If not already set, set additional manifest files to be applied after the deployment

export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}

# If not already set, set the neew Grafana admin password

export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}

########################################################################################################################

# Automated steps start here

########################################################################################################################

# Constants

GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller"

TMP_FOLDER="./tmp"

# Create a tmp folder for files modified during the deployment

TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"

TMP_LOGS_FOLDER="$TMP_FOLDER/logs"

echo "Deploying component and collecting environment variables..."

ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh

for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component..."

    IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG"

    IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')

    echo "  Building Docker image..."

    BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"

    if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then

        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"

    elif [ "$COMPONENT" == "pathcomp" ]; then

        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

        docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG"

        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

        docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

        # next command is redundant, but helpful to keep cache updated between rebuilds

        docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

    else

        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"

    fi

    if [ -n "$TFS_REGISTRY_IMAGES" ]; then

        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."

        if [ "$COMPONENT" == "pathcomp" ]; then

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"

            docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG"

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"

            docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG"

            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"

            docker push "$IMAGE_URL-frontend" > "$PUSH_LOG"

            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"

            docker push "$IMAGE_URL-backend" > "$PUSH_LOG"

        else

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"

            docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"

            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

        fi

    fi

    echo "  Adapting '$COMPONENT' manifest file..."

    MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"

    cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"

    if [ -n "$TFS_REGISTRY_IMAGES" ]; then

        # Registry is set

        if [ "$COMPONENT" == "pathcomp" ]; then

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST"

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"

        else

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"

        fi

    else

        # Registry is not set

        if [ "$COMPONENT" == "pathcomp" ]; then

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST"

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"

        else

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"

        fi

    fi

    # TODO: harmonize names of the monitoring component

    echo "  Deploying '$COMPONENT' component to Kubernetes..."

    DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"

    kubectl --namespace $TFS_K8S_NAMESPACE delete -f "$MANIFEST" > "$DEPLOY_LOG"

    kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"

    COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")

    kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"

    kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"

    echo "  Collecting env-vars for '$COMPONENT' component..."

    SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json)

    if [ -z "${SERVICE_DATA}" ]; then continue; fi

    # Env vars for service's host address

    SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP')

    if [ -z "${SERVICE_HOST}" ]; then continue; fi

    # TODO: remove previous value from file

    ENVVAR_HOST=$(echo "${COMPONENT}service_SERVICE_HOST" | tr '[:lower:]' '[:upper:]')

    echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT

    # Env vars for service's 'grpc' port (if any)

    SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port')

    if [ -n "${SERVICE_PORT_GRPC}" ]; then

        ENVVAR_PORT_GRPC=$(echo "${COMPONENT}service_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]')

        echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT

    fi

    # Env vars for service's 'http' port (if any)

    SERVICE_PORT_HTTP=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="http") | .port')

    if [ -n "${SERVICE_PORT_HTTP}" ]; then

        ENVVAR_PORT_HTTP=$(echo "${COMPONENT}service_SERVICE_PORT_HTTP" | tr '[:lower:]' '[:upper:]')

        echo "export ${ENVVAR_PORT_HTTP}=${SERVICE_PORT_HTTP}" >> $ENV_VARS_SCRIPT

    fi

    printf "\n"

done

# By now, leave this control here. Some component dependencies are not well handled

for COMPONENT in $TFS_COMPONENTS; do

    echo "Waiting for '$COMPONENT' component..."

    kubectl wait --namespace $TFS_K8S_NAMESPACE \

        --for='condition=available' --timeout=300s deployment/${COMPONENT}service

    printf "\n"

done

./show_deploy.sh

echo "Done!"

#!/bin/bash

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

########################################################################################################################

# Read deployment settings

########################################################################################################################

# If not already set, set the namespace where CockroackDB will be deployed.

export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}

# If not already set, set the database username to be used by Context.

export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"}

# If not already set, set the database user's password to be used by Context.

export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"}

# If not already set, set the database name to be used by Context.

export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}

# If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'.

# "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

# - If CRDB_DEPLOY_MODE is "single", CockroachDB is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If CRDB_DEPLOY_MODE is "cluster", CockroachDB is deployed in cluster mode, and an entire CockroachDB cluster

#   with 3 replicas and version v22.2 (set by default) will be deployed. It is convenient for production and

#   provides scalability features. If you are deploying for production, also read the following link providing

#   details on deploying CockroachDB for production environments:

#   Ref: https://www.cockroachlabs.com/docs/stable/recommended-production-settings.html

export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"}

# If not already set, disable flag for dropping database if exists.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_DATABASE will be dropped while

# checking/deploying CockroachDB.

export CRDB_DROP_DATABASE_IF_EXISTS=${CRDB_DROP_DATABASE_IF_EXISTS:-""}

# If not already set, disable flag for re-deploying CockroachDB from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# WARNING: THE REDEPLOY MIGHT TAKE FEW MINUTES TO COMPLETE GRACEFULLY IN CLUSTER MODE

# If CRDB_REDEPLOY is "YES", the database will be dropped while checking/deploying CockroachDB.

export CRDB_REDEPLOY=${CRDB_REDEPLOY:-""}

########################################################################################################################

# Automated steps start here

########################################################################################################################

# Constants

TMP_FOLDER="./tmp"

CRDB_MANIFESTS_PATH="manifests/cockroachdb"

# Create a tmp folder for files modified during the deployment

TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"

mkdir -p $TMP_MANIFESTS_FOLDER

TMP_LOGS_FOLDER="$TMP_FOLDER/logs"

mkdir -p $TMP_LOGS_FOLDER

CRDB_LOG_FILE="$TMP_LOGS_FOLDER/crdb_deploy.log"

function crdb_deploy_single() {

    echo "CockroachDB Namespace"

    echo ">>> Create CockroachDB Namespace (if missing)"

    kubectl create namespace ${CRDB_NAMESPACE}

    echo

    echo "CockroachDB (single-node)"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> CockroachDB is present; skipping step."

    else

        echo ">>> Deploy CockroachDB"

        cp "${CRDB_MANIFESTS_PATH}/single-node.yaml" "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml"

        sed -i "s/%CRDB_DATABASE%/${CRDB_DATABASE}/g" "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml"

        sed -i "s/%CRDB_USERNAME%/${CRDB_USERNAME}/g" "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml"

        sed -i "s/%CRDB_PASSWORD%/${CRDB_PASSWORD}/g" "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml"

        kubectl apply --namespace ${CRDB_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml"

        echo ">>> Waiting CockroachDB statefulset to be created..."

        while ! kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        # Wait for statefulset condition "Available=True" does not work

        # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error:

        #   "error: readyReplicas is not found"

        # Workaround: Check the pods are ready

        #echo ">>> CockroachDB statefulset created. Waiting for readiness condition..."

        #kubectl wait --namespace  ${CRDB_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/cockroachdb

        #kubectl wait --namespace ${CRDB_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \

        #    statefulset/cockroachdb

        echo ">>> CockroachDB statefulset created. Waiting CockroachDB pods to be created..."

        while ! kubectl get --namespace ${CRDB_NAMESPACE} pod/cockroachdb-0 &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        kubectl wait --namespace ${CRDB_NAMESPACE} --for=condition=Ready --timeout=300s pod/cockroachdb-0

    fi

    echo

    echo "CockroachDB Port Mapping"

    echo ">>> Expose CockroachDB SQL port (26257->26257)"

    CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

    PATCH='{"data": {"'${CRDB_SQL_PORT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_SQL_PORT}'"}}'

    kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}"

    PORT_MAP='{"containerPort": '${CRDB_SQL_PORT}', "hostPort": '${CRDB_SQL_PORT}'}'

    CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}'

    PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}'

    kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}"

    echo

    echo ">>> Expose CockroachDB HTTP Mgmt GUI port (8080->8081)"

    CRDB_GUI_PORT_EXT="8081"

    CRDB_GUI_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}')

    PATCH='{"data": {"'${CRDB_GUI_PORT_EXT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_GUI_PORT}'"}}'

    kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}"

    PORT_MAP='{"containerPort": '${CRDB_GUI_PORT_EXT}', "hostPort": '${CRDB_GUI_PORT_EXT}'}'

    CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}'

    PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}'

    kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}"

    echo

}

function crdb_undeploy_single() {

    echo "CockroachDB"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> Undeploy CockroachDB"

        kubectl delete --namespace ${CRDB_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/crdb_single_node.yaml" --ignore-not-found

    else

        echo ">>> CockroachDB is not present; skipping step."

    fi

    echo

    echo "CockroachDB Namespace"

    echo ">>> Delete CockroachDB Namespace (if exists)"

    echo "NOTE: this step might take few minutes to complete!"

    kubectl delete namespace ${CRDB_NAMESPACE} --ignore-not-found

    echo

}

function crdb_drop_database_single() {

    echo "Drop database if exists"

    CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@cockroachdb-0:${CRDB_SQL_PORT}/defaultdb?sslmode=require"

    kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \

        ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \

        --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};"

    echo

}

function crdb_deploy_cluster() {

    echo "Cockroach Operator CRDs"

    echo ">>> Apply Cockroach Operator CRDs (if they are missing)"

    cp "${CRDB_MANIFESTS_PATH}/crds.yaml" "${TMP_MANIFESTS_FOLDER}/crdb_crds.yaml"

    kubectl apply -f "${TMP_MANIFESTS_FOLDER}/crdb_crds.yaml"

    echo

    echo "Cockroach Operator"

    echo ">>> Checking if Cockroach Operator is deployed..."

    if kubectl get --namespace cockroach-operator-system deployment/cockroach-operator-manager &> /dev/null; then

        echo ">>> Cockroach Operator is present; skipping step."

    else

        echo ">>> Deploy Cockroach Operator"

        sed "s/%TFS_CRDB_NAMESPACE%/${CRDB_NAMESPACE}/g" "${CRDB_MANIFESTS_PATH}/operator.yaml" \

            > "${TMP_MANIFESTS_FOLDER}/crdb_operator.yaml"

        kubectl apply -f "${TMP_MANIFESTS_FOLDER}/crdb_operator.yaml"

        kubectl wait --namespace cockroach-operator-system --for=condition=Available=True --timeout=300s \

            deployment/cockroach-operator-manager

        #kubectl wait --namespace cockroach-operator-system --for=jsonpath='{.status.readyReplicas}'=1 --timeout=300s \

        #    deployment/cockroach-operator-manager

        echo ">>> Waiting for Cockroach Operator Webhock service..."

        while ! kubectl get service cockroach-operator-webhook-service --namespace cockroach-operator-system &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        WEBHOOK_SERVICE_DATA=$(kubectl get service cockroach-operator-webhook-service --namespace cockroach-operator-system -o json)

        WEBHOOK_SERVICE_HOST=$(echo ${WEBHOOK_SERVICE_DATA} | jq -r '.spec.clusterIP')

        WEBHOOK_SERVICE_PORT=$(echo ${WEBHOOK_SERVICE_DATA} | jq -r '.spec.ports[] | select(.targetPort==9443) | .port')

        WEBHOOK_URL="https://${WEBHOOK_SERVICE_HOST}:${WEBHOOK_SERVICE_PORT}/mutate-crdb-cockroachlabs-com-v1alpha1-crdbcluster?timeout=10s"

        while ! curl --insecure --header 'Content-Type: application/json' ${WEBHOOK_URL} &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

    fi

    echo

    echo "CockroachDB Namespace"

    echo ">>> Create CockroachDB Namespace (if missing)"

    kubectl create namespace ${CRDB_NAMESPACE}

    echo

    echo "CockroachDB"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> CockroachDB is present; skipping step."

    else

        echo ">>> Deploy CockroachDB"

        cp "${CRDB_MANIFESTS_PATH}/cluster.yaml" "${TMP_MANIFESTS_FOLDER}/crdb_cluster.yaml"

        kubectl apply --namespace ${CRDB_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/crdb_cluster.yaml"

        echo ">>> Waiting CockroachDB statefulset to be created..."

        while ! kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        # Wait for statefulset condition "Available=True" does not work

        # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error:

        #   "error: readyReplicas is not found"

        # Workaround: Check the pods are ready

        #echo ">>> CockroachDB statefulset created. Waiting for readiness condition..."

        #kubectl wait --namespace  ${CRDB_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/cockroachdb

        #kubectl wait --namespace ${CRDB_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \

        #    statefulset/cockroachdb

        echo ">>> CockroachDB statefulset created. Waiting CockroachDB pods to be created..."

        while ! kubectl get --namespace ${CRDB_NAMESPACE} pod/cockroachdb-0 &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        while ! kubectl get --namespace ${CRDB_NAMESPACE} pod/cockroachdb-1 &> /dev/null; do

            printf "%c" "."

            sleep 1