deploy/kafka.sh

+#!/bin/bash
+# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+########################################################################################################################
+# Read deployment settings
+########################################################################################################################
+
+# If not already set, set the namespace where Apache Kafka will be deployed.
+export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"}
+
+
+########################################################################################################################
+# Automated steps start here
+########################################################################################################################
+
+# Constants
+TMP_FOLDER="./tmp"
+KFK_MANIFESTS_PATH="manifests/kafka"
+KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml"
+KFK_MANIFEST="02-kafka.yaml"
+
+# Create a tmp folder for files modified during the deployment
+TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests"
+mkdir -p ${TMP_MANIFESTS_FOLDER}
+
+# copy zookeeper and kafka manifest files to temporary manifest location
+cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
+cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"
+
+kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found
+kubectl create namespace ${KFK_NAMESPACE}
+# sleep 2
+# echo "----"
+
+# Kafka zookeeper service should be deployed before the kafka service
+kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}"
+# kubectl get services --namespace ${KFK_NAMESPACE}
+# echo "----"
+
+KFK_ZOOKEEPER_SERVICE="zookeeper-service"    # this command may be replaced with command to get service name automatically
+KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}')
+# echo $KFK_ZOOKEEPER_IP
+# echo "----"
+
+# Kafka service should be deployed after the zookeeper service
+sed -i "s/<ZOOKEEPER_INTERNAL_IP>/${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
+# echo "----"
+kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST"
+sleep 5
+kubectl --namespace ${KFK_NAMESPACE} get pods
+
+echo "--- Kafka service deployed sucessfully ---"
\ No newline at end of file

manifests/kafka/01-zookeeper.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: zookeeper-service
+  name: zookeeper-service
+  namespace: kafka
+spec:
+  type: NodePort
+  ports:
+    - name: zookeeper-port
+      port: 2181
+      nodePort: 30181
+      targetPort: 2181
+  selector:
+    app: zookeeper
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: zookeeper
+  name: zookeeper
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: zookeeper
+  template:
+    metadata:
+      labels:
+        app: zookeeper
+    spec:
+      containers:
+        - image: wurstmeister/zookeeper
+          imagePullPolicy: IfNotPresent
+          name: zookeeper
+          ports:
+            - containerPort: 2181
\ No newline at end of file

manifests/kafka/02-kafka.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kafka-broker
+  name: kafka-service
+  namespace: kafka
+spec:
+  ports:
+  - port: 9092
+  selector:
+    app: kafka-broker
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: kafka-broker
+  name: kafka-broker
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-broker
+  template:
+    metadata:
+      labels:
+        app: kafka-broker
+    spec:
+      hostname: kafka-broker
+      containers:
+      - env:
+        - name: KAFKA_BROKER_ID
+          value: "1"
+        - name: KAFKA_ZOOKEEPER_CONNECT
+          value: <ZOOKEEPER_INTERNAL_IP>:2181
+        - name: KAFKA_LISTENERS
+          value: PLAINTEXT://:9092
+        - name: KAFKA_ADVERTISED_LISTENERS
+          value: PLAINTEXT://localhost:9092
+        image: wurstmeister/kafka
+        imagePullPolicy: IfNotPresent
+        name: kafka-broker
+        ports:
+          - containerPort: 9092
\ No newline at end of file

my_deploy.sh

@@ -154,3 +154,10 @@ export PROM_EXT_PORT_HTTP="9090"
 
 # Set the external port Grafana HTTP Dashboards will be exposed to.
 export GRAF_EXT_PORT_HTTP="3000"
+
+
+# ----- Apache Kafka -----------------------------------------------------------
+
+# Set the namespace where Apache Kafka will be deployed.
+export KFK_NAMESPACE="kafka"
+

src/kpi_manager/tests/test_unitary.py

@@ -239,6 +239,7 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab
 # Prepare Environment, should be the first test
 ##################################################
 
+# ERROR on this test --- 
 def test_prepare_environment(
     context_client : ContextClient,                 # pylint: disable=redefined-outer-name,unused-argument
 ):

src/monitoring/requirements.in

@@ -32,3 +32,15 @@ requests==2.27.1
 xmltodict==0.12.0
 questdb==1.0.1
 psycopg2-binary==2.9.3
+coverage==6.3
+grpcio==1.47.*
+grpcio-health-checking==1.47.*
+grpcio-tools==1.47.*
+grpclib==0.4.4
+prettytable==3.5.0
+prometheus-client==0.13.0
+protobuf==3.20.*
+pytest==6.2.5
+pytest-benchmark==3.4.1
+python-dateutil==2.8.2
+pytest-depends==1.0.1