From 00a0bcc447eded1e1f6f6586bda4878f036a1f47 Mon Sep 17 00:00:00 2001
From: Waleed Akbar <wakbar@cttc.es>
Date: Tue, 30 Jul 2024 14:14:18 +0000
Subject: [PATCH] Kafka deployment script is integrated with TFS deplyment
script.
- Kafka env variables are created in my_deply.sh, kafka.sh and all.sh
---
deploy/all.sh | 3 ++
deploy/kafka.sh | 83 ++++++++++++++++++++++++++++++-------------------
deploy/tfs.sh | 20 +++++++++++-
3 files changed, 73 insertions(+), 33 deletions(-)
diff --git a/deploy/all.sh b/deploy/all.sh
index f93cd92ac..e9b33b469 100755
--- a/deploy/all.sh
+++ b/deploy/all.sh
@@ -215,6 +215,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
# Deploy QuestDB
./deploy/qdb.sh
+# Deploy Apache Kafka
+./deploy/kafka.sh
+
# Expose Dashboard
./deploy/expose_dashboard.sh
diff --git a/deploy/kafka.sh b/deploy/kafka.sh
index 4a91bfc9e..b2f2f1f9e 100755
--- a/deploy/kafka.sh
+++ b/deploy/kafka.sh
@@ -20,50 +20,69 @@
# If not already set, set the namespace where Apache Kafka will be deployed.
export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"}
+# If not already set, set the port Apache Kafka server will be exposed to.
+export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"}
+
+# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost.
+export KFK_REDEPLOY=${KFK_REDEPLOY:-""}
+
########################################################################################################################
# Automated steps start here
########################################################################################################################
-# Constants
-TMP_FOLDER="./tmp"
-KFK_MANIFESTS_PATH="manifests/kafka"
-KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml"
-KFK_MANIFEST="02-kafka.yaml"
+ # Constants
+ TMP_FOLDER="./tmp"
+ KFK_MANIFESTS_PATH="manifests/kafka"
+ KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml"
+ KFK_MANIFEST="02-kafka.yaml"
+
+ # Create a tmp folder for files modified during the deployment
+ TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests"
+ mkdir -p ${TMP_MANIFESTS_FOLDER}
-# Create a tmp folder for files modified during the deployment
-TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests"
-mkdir -p ${TMP_MANIFESTS_FOLDER}
+function kafka_deploy() {
+ # copy zookeeper and kafka manifest files to temporary manifest location
+ cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
+ cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"
-# copy zookeeper and kafka manifest files to temporary manifest location
-cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
-cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"
+ # echo "Apache Kafka Namespace"
+ echo ">>> Delete Apache Kafka Namespace"
+ kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found
-echo "Apache Kafka Namespace"
-echo ">>> Delete Apache Kafka Namespace"
-kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found
+ echo ">>> Create Apache Kafka Namespace"
+ kubectl create namespace ${KFK_NAMESPACE}
-echo ">>> Create Apache Kafka Namespace"
-kubectl create namespace ${KFK_NAMESPACE}
+ # echo ">>> Deplying Apache Kafka Zookeeper"
+ # Kafka zookeeper service should be deployed before the kafka service
+ kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
-echo ">>> Deplying Apache Kafka Zookeeper"
-# Kafka zookeeper service should be deployed before the kafka service
-kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
+ KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically
+ KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}')
-KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically
-KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}')
+ # Kafka service should be deployed after the zookeeper service
+ sed -i "s/<ZOOKEEPER_INTERNAL_IP>/${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
-# Kafka service should be deployed after the zookeeper service
-sed -i "s/<ZOOKEEPER_INTERNAL_IP>/${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
+ # echo ">>> Deploying Apache Kafka Broker"
+ kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
-echo ">>> Deploying Apache Kafka Broker"
-kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
+ # echo ">>> Verifing Apache Kafka deployment"
+ sleep 5
+ # KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods)
+ # if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then
+ # echo "Deployment Error: \n $KFK_PODS_STATUS"
+ # else
+ # echo "$KFK_PODS_STATUS"
+ # fi
+}
-echo ">>> Verifing Apache Kafka deployment"
-sleep 10
-KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods)
-if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then
- echo "Deployment Error: \n $KFK_PODS_STATUS"
+echo "Apache Kafka"
+echo ">>> Checking if Apache Kafka is deployed ... "
+if [ "$KFK_REDEPLOY" == "YES" ]; then
+ kafka_deploy
+elif kubectl get --namespace ${KFK_NAMESPACE} deployments.apps &> /dev/null; then
+ echo ">>> Apache Kafka already present; skipping step..."
else
- echo "$KFK_PODS_STATUS"
-fi
\ No newline at end of file
+ kafka_deploy
+fi
+echo
diff --git a/deploy/tfs.sh b/deploy/tfs.sh
index 62f36a2c1..d92d789e3 100755
--- a/deploy/tfs.sh
+++ b/deploy/tfs.sh
@@ -115,6 +115,17 @@ export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"}
export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
+# ----- Apache Kafka ------------------------------------------------------
+
+# If not already set, set the namespace where Apache Kafka will be deployed.
+export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"}
+
+# If not already set, set the port Apache Kafka server will be exposed to.
+export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"}
+
+# If not already set, if flag is YES, Apache Kafka will be redeployed and topic will be lost.
+export KFK_REDEPLOY=${KFK_REDEPLOY:-""}
+
########################################################################################################################
# Automated steps start here
########################################################################################################################
@@ -147,7 +158,7 @@ kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type=
--from-literal=CRDB_SSLMODE=require
printf "\n"
-echo "Create secret with CockroachDB data for KPI Management"
+echo "Create secret with CockroachDB data for KPI Management microservices"
CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
CRDB_DATABASE_KPI_MGMT="tfs_kpi_mgmt" # TODO: change by specific configurable environment variable
kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
@@ -159,6 +170,13 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t
--from-literal=CRDB_SSLMODE=require
printf "\n"
+echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices"
+KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')
+kubectl create secret generic kfk-kpi-data --namespace ${KFK_NAMESPACE} --type='Opaque' \
+ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
+ --from-literal=KFK_SERVER_PORT=${KFK_NAMESPACE}
+printf "\n"
+
echo "Create secret with NATS data"
NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}')
if [ -z "$NATS_CLIENT_PORT" ]; then
--
GitLab