Commit 865c004c authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'develop' of ssh://gifrerenom_labs.etsi.org/tfs/controller into... 

Merge branch 'develop' of ssh://gifrerenom_labs.etsi.org/tfs/controller into feat/158-fixing-containerlab-lab-example
parents 67b34988 68e713f6

.gitignore

+3 −0
Original line number Diff line number Diff line
@@ -176,3 +176,6 @@ libyang/ 


# Other logs

**/logs/*.log.*



# PySpark checkpoints

src/analytics/.spark/*

.gitlab-ci.yml

+4 −1
Original line number Diff line number Diff line
@@ -45,6 +45,9 @@ include: 
  #- local: '/src/dlt/.gitlab-ci.yml'

  - local: '/src/load_generator/.gitlab-ci.yml'

  - local: '/src/bgpls_speaker/.gitlab-ci.yml'



  - local: '/src/kpi_manager/.gitlab-ci.yml'

  - local: '/src/kpi_value_api/.gitlab-ci.yml'

  - local: '/src/kpi_value_writer/.gitlab-ci.yml'

  - local: '/src/telemetry/.gitlab-ci.yml'

  # This should be last one: end-to-end integration tests

  - local: '/src/tests/.gitlab-ci.yml'

deploy/all.sh

+54 −5
Original line number Diff line number Diff line
@@ -27,7 +27,47 @@ export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"} 


# If not already set, set the list of components, separated by spaces, you want to build images for, and deploy.

# By default, only basic components are deployed

export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device ztp monitoring pathcomp service slice nbi webui load_generator"}

export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice nbi webui load_generator"}



# Uncomment to activate Monitoring (old)

#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"



# Uncomment to activate Monitoring Framework (new)

#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics"



# Uncomment to activate BGP-LS Speaker

#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker"



# Uncomment to activate Optical Controller

#   To manage optical connections, "service" requires "opticalcontroller" to be deployed

#   before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the

#   "opticalcontroller" only if "service" is already in TFS_COMPONENTS, and re-export it.

#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then

#    BEFORE="${TFS_COMPONENTS% service*}"

#    AFTER="${TFS_COMPONENTS#* service}"

#    export TFS_COMPONENTS="${BEFORE} opticalcontroller service ${AFTER}"

#fi



# Uncomment to activate ZTP

#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp"



# Uncomment to activate Policy Manager

#export TFS_COMPONENTS="${TFS_COMPONENTS} policy"



# Uncomment to activate Optical CyberSecurity

#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager"



# Uncomment to activate L3 CyberSecurity

#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector"



# Uncomment to activate TE

#export TFS_COMPONENTS="${TFS_COMPONENTS} te"



# Uncomment to activate Forecaster

#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster"



# Uncomment to activate E2E Orchestrator

#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator"



# If not already set, set the tag you want to use for your images.

export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}
@@ -67,8 +107,6 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} 
export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}



# If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'.

# "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

# - If CRDB_DEPLOY_MODE is "single", CockroachDB is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If CRDB_DEPLOY_MODE is "cluster", CockroachDB is deployed in cluster mode, and an entire CockroachDB cluster
@@ -80,7 +118,7 @@ export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"} 


# If not already set, disable flag for dropping database, if it exists.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_DATABASE will be dropped while

# checking/deploying CockroachDB.

export CRDB_DROP_DATABASE_IF_EXISTS=${CRDB_DROP_DATABASE_IF_EXISTS:-""}
@@ -102,6 +140,14 @@ export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} 
# If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to.

export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"}



# If not already set, set NATS installation mode. Accepted values are: 'single' and 'cluster'.

# - If NATS_DEPLOY_MODE is "single", NATS is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If NATS_DEPLOY_MODE is "cluster", NATS is deployed in cluster mode, and an entire NATS cluster

#   with 3 replicas (set by default) will be deployed. It is convenient for production and

#   provides scalability features.

export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"}



# If not already set, disable flag for re-deploying NATS from scratch.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION!

# If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS.
@@ -169,6 +215,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} 
# Deploy QuestDB

./deploy/qdb.sh



# Deploy Apache Kafka

./deploy/kafka.sh



# Expose Dashboard

./deploy/expose_dashboard.sh

deploy/crdb.sh

+5 −7
Original line number Diff line number Diff line
@@ -37,8 +37,6 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} 
export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}



# If not already set, set CockroachDB installation mode. Accepted values are: 'single' and 'cluster'.

# "YES", the database pointed by variable CRDB_NAMESPACE will be dropped while

# checking/deploying CockroachDB.

# - If CRDB_DEPLOY_MODE is "single", CockroachDB is deployed in single node mode. It is convenient for

#   development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS.

# - If CRDB_DEPLOY_MODE is "cluster", CockroachDB is deployed in cluster mode, and an entire CockroachDB cluster
@@ -48,7 +46,7 @@ export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"} 
#   Ref: https://www.cockroachlabs.com/docs/stable/recommended-production-settings.html

export CRDB_DEPLOY_MODE=${CRDB_DEPLOY_MODE:-"single"}



# If not already set, disable flag for dropping database if exists.

# If not already set, disable flag for dropping database, if it exists.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE DATABASE INFORMATION!

# If CRDB_DROP_DATABASE_IF_EXISTS is "YES", the database pointed by variable CRDB_DATABASE will be dropped while

# checking/deploying CockroachDB.
@@ -79,7 +77,7 @@ function crdb_deploy_single() { 
    kubectl create namespace ${CRDB_NAMESPACE}

    echo



    echo "CockroachDB (single-node)"

    echo "CockroachDB (single-mode)"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> CockroachDB is present; skipping step."
@@ -139,7 +137,7 @@ function crdb_deploy_single() { 
}



function crdb_undeploy_single() {

    echo "CockroachDB"

    echo "CockroachDB (single-mode)"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> Undeploy CockroachDB"
@@ -223,7 +221,7 @@ function crdb_deploy_cluster() { 
    kubectl create namespace ${CRDB_NAMESPACE}

    echo



    echo "CockroachDB"

    echo "CockroachDB (cluster-mode)"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> CockroachDB is present; skipping step."
@@ -319,7 +317,7 @@ function crdb_undeploy_cluster() { 
    fi

    echo



    echo "CockroachDB"

    echo "CockroachDB (cluster-mode)"

    echo ">>> Checking if CockroachDB is deployed..."

    if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then

        echo ">>> Undeploy CockroachDB"

deploy/kafka.sh

0 → 100755
+90 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



########################################################################################################################

# Read deployment settings

########################################################################################################################



# If not already set, set the namespace where Apache Kafka will be deployed.

export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"}



# If not already set, set the port Apache Kafka server will be exposed to.

export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"}



# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost.

export KFK_REDEPLOY=${KFK_REDEPLOY:-""}





########################################################################################################################

# Automated steps start here

########################################################################################################################



    # Constants

    TMP_FOLDER="./tmp"

    KFK_MANIFESTS_PATH="manifests/kafka"

    KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml"

    KFK_MANIFEST="02-kafka.yaml"



    # Create a tmp folder for files modified during the deployment

    TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests"

    mkdir -p ${TMP_MANIFESTS_FOLDER}



function kafka_deploy() {

     # copy zookeeper and kafka manifest files to temporary manifest location

    cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"

    cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"



    # echo "Apache Kafka Namespace"

    echo ">>> Delete Apache Kafka Namespace"

    kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found



    echo ">>> Create Apache Kafka Namespace"

    kubectl create namespace ${KFK_NAMESPACE}



    # echo ">>> Deplying Apache Kafka Zookeeper"

    # Kafka zookeeper service should be deployed before the kafka service

    kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"



    KFK_ZOOKEEPER_SERVICE="zookeeper-service"    # this command may be replaced with command to extract service name automatically

    KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}')



    # Kafka service should be deployed after the zookeeper service

    sed -i "s/<ZOOKEEPER_INTERNAL_IP>/${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"



    # echo ">>> Deploying Apache Kafka Broker"

    kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"



    # echo ">>> Verifing Apache Kafka deployment"

    sleep 5

    # KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods)

    # if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then

    #     echo "Deployment Error: \n $KFK_PODS_STATUS"

    # else

    #     echo "$KFK_PODS_STATUS"

    # fi

}



echo "Apache Kafka"

echo ">>> Checking if Apache Kafka is deployed ... "

if [ "$KFK_REDEPLOY" == "YES" ]; then

    echo ">>> Redeploying kafka namespace"

    kafka_deploy

elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then

    echo ">>> Apache Kafka already present; skipping step." 

else

    echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace"

    kafka_deploy

fi

echo