Commit 903748da authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Deploy scripts: 

- updated deploy scripts to include CockroachDB and NATS
- reorganized deploy scripts in a new "deploy" folder
parent 6733b6c1

deploy/all.sh

0 → 100644
+118 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





########################################################################################################################

# Read deployment settings

########################################################################################################################



# If not already set, set the URL of the Docker registry where the images will be uploaded to.

# By default, assume internal MicroK8s registry is used.

export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}



# If not already set, set the list of components you want to build images for, and deploy.

# By default, only basic components are deployed

export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"}



# If not already set, set the tag you want to use for your images.

export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}



# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}



# If not already set, set additional manifest files to be applied after the deployment

export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}



# If not already set, set the new Grafana admin password

export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}



# If not already set, disable skip-build flag.

# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.

export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}



# If not already set, set the namespace where CockroackDB will be deployed.

export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}



# If not already set, set the database username to be used by Context.

export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"}



# If not already set, set the database user's password to be used by Context.

export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"}



# If not already set, set the database name to be used by Context.

export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}



# If not already set, set the name of the secret where CockroachDB data and credentials will be stored.

export CRDB_SECRET_NAME=${CRDB_SECRET_NAME:-"crdb-data"}



# If not already set, set the namespace where the secret containing CockroachDB data and credentials will be stored.

export CRDB_SECRET_NAMESPACE=${CRDB_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}}



# If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'.

# "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

# - If CRDB_DEPLOY_MODE is "single", CockroachDB is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If CRDB_DEPLOY_MODE is "cluster", CockroachDB is deployed in cluster mode, and an entire CockroachDB cluster

#   with 3 replicas and version v22.2 (set by default) will be deployed. It is convenient for production and

#   provides scalability features. If you are deploying for production, also read the following link providing

#   details on deploying CockroachDB for production environments:

#   Ref: https://www.cockroachlabs.com/docs/stable/recommended-production-settings.html

export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"}



# If not already set, disable flag for dropping database if exists.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

export CRDB_DROP_DATABASE_IF_EXISTS=${CRDB_DROP_DATABASE_IF_EXISTS:-""}



# If not already set, disable flag for re-deploying CockroachDB from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# WARNING: THE REDEPLOY MIGHT TAKE FEW MINUTES TO COMPLETE GRACEFULLY IN CLUSTER MODE

# If CRDB_REDEPLOY is "YES", the database will be dropped while checking/deploying CockroachDB.

export CRDB_REDEPLOY=${CRDB_REDEPLOY:-""}



# If not already set, set the namespace where NATS will be deployed.

export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}



# If not already set, set the name of the secret where NATS data and credentials will be stored.

export NATS_SECRET_NAME=${NATS_SECRET_NAME:-"nats-data"}



# If not already set, set the namespace where the secret containing NATS data and credentials will be stored.

export NATS_SECRET_NAMESPACE=${NATS_SECRET_NAMESPACE:-${TFS_K8S_NAMESPACE}}



# If not already set, disable flag for re-deploying NATS from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION!

# If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS.

export NATS_REDEPLOY=${NATS_REDEPLOY:-""}





########################################################################################################################

# Automated steps start here

########################################################################################################################



# Deploy CockroachDB

./deploy/deploy_crdb.sh



# Deploy NATS

./deploy/deploy_nats.sh



# Deploy TFS

./deploy/deploy_tfs.sh



# Show deploy summary

./show_deploy.sh



echo "Done!"

deploy/crdb.sh

0 → 100755
+396 −0

File added.

Preview size limit exceeded, changes collapsed.

deploy_component.sh→deploy/deploy_component.sh

+7 −8
Original line number Diff line number Diff line
@@ -18,10 +18,9 @@ 
# Read deployment settings

########################################################################################################################



# If not already set, set the URL of your local Docker registry where the images will be uploaded to.

# Leave it blank if you do not want to use any Docker registry.

export TFS_REGISTRY_IMAGE=${TFS_REGISTRY_IMAGE:-""}

#export TFS_REGISTRY_IMAGE="http://my-container-registry.local/"

# If not already set, set the URL of the Docker registry where the images will be uploaded to.

# By default, assume internal MicroK8s registry is used.

export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}



TFS_COMPONENTS=$1
@@ -55,7 +54,7 @@ ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh 
for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component..."

    IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG"

    IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')

    IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')



    echo "  Building Docker image..."

    BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
@@ -74,8 +73,8 @@ for COMPONENT in $TFS_COMPONENTS; do 
        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"

    fi



    if [ -n "$TFS_REGISTRY_IMAGE" ]; then

        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGE'..."

    if [ -n "$TFS_REGISTRY_IMAGES" ]; then

        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."



        if [ "$COMPONENT" == "pathcomp" ]; then

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
@@ -102,7 +101,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
    MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"

    cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"



    if [ -n "$TFS_REGISTRY_IMAGE" ]; then

    if [ -n "$TFS_REGISTRY_IMAGES" ]; then

        # Registry is set

        if [ "$COMPONENT" == "pathcomp" ]; then

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)

deploy/nats.sh

0 → 100755
+144 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



########################################################################################################################

# Read deployment settings

########################################################################################################################



# If not already set, set the namespace where NATS will be deployed.

export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}



# If not already set, set the name of the secret where NATS data and credentials will be stored.

export NATS_SECRET_NAME=${NATS_SECRET_NAME:-"nats-data"}



# If not already set, set the namespace where the secret containing NATS data and credentials will be stored.

export NATS_SECRET_NAMESPACE=${NATS_SECRET_NAMESPACE:-"tfs"}



# If not already set, disable flag for re-deploying NATS from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION!

# If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS.

export NATS_REDEPLOY=${NATS_REDEPLOY:-""}



########################################################################################################################

# Automated steps start here

########################################################################################################################



# Constants

TMP_FOLDER="./tmp"

NATS_MANIFESTS_PATH="manifests/nats"



# Create a tmp folder for files modified during the deployment

TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"

mkdir -p $TMP_MANIFESTS_FOLDER



function nats_deploy_single() {

    echo "NATS Namespace"

    echo ">>> Create NATS Namespace (if missing)"

    kubectl create namespace ${NATS_NAMESPACE}

    echo



    echo "Add NATS Helm Chart"

    helm3 repo add nats https://nats-io.github.io/k8s/helm/charts/

    echo



    echo "Install NATS (single-node)"

    echo ">>> Checking if NATS is deployed..."

    if kubectl get --namespace ${NATS_NAMESPACE} statefulset/nats &> /dev/null; then

        echo ">>> NATS is present; skipping step."

    else

        echo ">>> Deploy NATS"

        helm3 install nats nats/nats --namespace ${NATS_NAMESPACE} --set nats.image.tag=2.9-alpine



        echo ">>> Waiting NATS statefulset to be created..."

        while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/nats &> /dev/null; do

            printf "%c" "."

            sleep 1

        done



        # Wait for statefulset condition "Available=True" does not work

        # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error:

        #   "error: readyReplicas is not found"

        # Workaround: Check the pods are ready

        #echo ">>> NATS statefulset created. Waiting for readiness condition..."

        #kubectl wait --namespace  ${NATS_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/nats

        #kubectl wait --namespace ${NATS_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \

        #    statefulset/nats

        echo ">>> NATS statefulset created. Waiting NATS pods to be created..."

        while ! kubectl get --namespace ${NATS_NAMESPACE} pod/nats-0 &> /dev/null; do

            printf "%c" "."

            sleep 1

        done

        kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/nats-0

    fi

    echo



    echo "NATS Port Mapping"

    echo ">>> Expose NATS Client port (4222)"

    NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service nats -o 'jsonpath={.spec.ports[?(@.name=="client")].port}')

    PATCH='{"data": {"'${NATS_CLIENT_PORT}'": "'${NATS_NAMESPACE}'/nats:'${NATS_CLIENT_PORT}'"}}'

    kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}"



    PORT_MAP='{"containerPort": '${NATS_CLIENT_PORT}', "hostPort": '${NATS_CLIENT_PORT}'}'

    CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}'

    PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}'

    kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}"

    echo



    echo ">>> Expose NATS HTTP Mgmt GUI port (8222)"

    NATS_GUI_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service nats -o 'jsonpath={.spec.ports[?(@.name=="monitor")].port}')

    PATCH='{"data": {"'${NATS_GUI_PORT}'": "'${NATS_NAMESPACE}'/nats:'${NATS_GUI_PORT}'"}}'

    kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}"



    PORT_MAP='{"containerPort": '${NATS_GUI_PORT}', "hostPort": '${NATS_GUI_PORT}'}'

    CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}'

    PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}'

    kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}"

    echo



    echo "Create secret with NATS data"

    kubectl create secret generic ${NATS_SECRET_NAME} --namespace ${NATS_SECRET_NAMESPACE} --type='Opaque' \

        --from-literal=namespace=${NATS_NAMESPACE} \

        --from-literal=client_port=${NATS_CLIENT_PORT} \

        --from-literal=gui_port=${NATS_GUI_PORT}



    kubectl get all --all-namespaces

}



function nats_undeploy_single() {

    echo "Delete secret with NATS data"

    kubectl delete secret ${NATS_SECRET_NAME} --namespace ${NATS_SECRET_NAMESPACE} --ignore-not-found

    echo



    echo "NATS"

    echo ">>> Checking if NATS is deployed..."

    if kubectl get --namespace ${NATS_NAMESPACE} statefulset/nats &> /dev/null; then

        echo ">>> Undeploy NATS"

        helm3 uninstall --namespace ${NATS_NAMESPACE} nats

    else

        echo ">>> NATS is not present; skipping step."

    fi

    echo



    echo "NATS Namespace"

    echo ">>> Delete NATS Namespace (if exists)"

    kubectl delete namespace ${NATS_NAMESPACE} --ignore-not-found

    echo

}



if [ "$NATS_REDEPLOY" == "YES" ]; then

    nats_undeploy_single

fi



nats_deploy_single

show_deploy.sh→deploy/show.sh

+0 −0

File moved.

View file