Merge branch 'feat/331-bitnami-kafka-moved-to-bitnamilegacy-kafka' into 'develop' (eddce1f8) · Commits · TFS / controller

manifests/kafka/single-node.yaml

+1 −9

Original line number	Diff line number	Diff line
		@@ -60,7 +60,7 @@ spec:
		restartPolicy: Always
		containers:
		- name: kafka
		image: bitnami/kafka:latest
		image: bitnamilegacy/kafka:latest
		imagePullPolicy: IfNotPresent
		ports:
		- name: clients
		@@ -68,14 +68,6 @@ spec:
		- name: control-plane
		containerPort: 9093
		env:
		#- name: KAFKA_BROKER_ID
		# value: "1"
		#- name: KAFKA_ZOOKEEPER_CONNECT
		# value: zookeeper-service.<KAFKA_NAMESPACE>.svc.cluster.local:2181
		#- name: KAFKA_LISTENERS
		# value: PLAINTEXT://:9092
		#- name: KAFKA_ADVERTISED_LISTENERS
		# value: PLAINTEXT://kafka-service.<KAFKA_NAMESPACE>.svc.cluster.local:9092
		- name: KAFKA_CFG_NODE_ID
		value: "1"
		- name: KAFKA_CFG_PROCESS_ROLES

scripts/run_tests_locally-nbi-all.sh

+2 −2

Original line number	Diff line number	Diff line
		@@ -18,7 +18,7 @@ docker ps -aq \| xargs -r docker rm -f
		docker network rm teraflowbridge \|\| true
		docker container prune -f

		docker pull "bitnami/kafka:latest"
		docker pull "bitnamilegacy/kafka:latest"
		docker buildx build -t "mock_tfs_nbi_dependencies:test" -f ./src/tests/tools/mock_tfs_nbi_dependencies/Dockerfile .
		docker buildx build -t "nbi:latest" -f ./src/nbi/Dockerfile .
		docker image prune --force
		@@ -32,7 +32,7 @@ docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 \
		--env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT \
		--env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
		--env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 \
		bitnami/kafka:latest
		bitnamilegacy/kafka:latest

		echo "Waiting for Kafka to be ready..."
		while ! docker logs kafka 2>&1 \| grep -q 'Kafka Server started'; do

src/analytics/.gitlab-ci.yml

+5 −32

Original line number	Diff line number	Diff line
		@@ -60,24 +60,12 @@ unit_test analytics-backend:
		- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		- if docker network list \| grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
		- if docker container ls \| grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi
		#- if docker container ls \| grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi
		# - if docker container ls \| grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi
		- if docker container ls \| grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi
		- if docker container ls \| grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi
		- docker container prune -f
		script:
		- docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG"
		#- docker pull "bitnami/zookeeper:latest"
		- docker pull "bitnami/kafka:latest"
		#- >
		# docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181
		# --env ALLOW_ANONYMOUS_LOGIN=yes
		# bitnami/zookeeper:latest
		#- sleep 10 # Wait for Zookeeper to start
		#- >
		# docker run --name kafka -d --network=teraflowbridge -p 9092:9092
		# --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
		# --env ALLOW_PLAINTEXT_LISTENER=yes
		# bitnami/kafka:latest
		- docker pull "bitnamilegacy/kafka:latest"
		- >
		docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093
		--env KAFKA_CFG_NODE_ID=1
		@@ -86,7 +74,7 @@ unit_test analytics-backend:
		--env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
		--env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
		--env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
		bitnami/kafka:latest
		bitnamilegacy/kafka:latest
		- while ! docker logs kafka 2>&1 \| grep -q 'Kafka Server started'; do sleep 1; done
		- sleep 5 # Give extra time to Kafka to get stabilized
		- docker inspect kafka --format "{{.NetworkSettings.Networks}}"
		@@ -109,7 +97,6 @@ unit_test analytics-backend:
		after_script:
		- docker rm -f ${IMAGE_NAME}-backend
		- docker rm -f kafka
		#- docker rm -f zookeeper
		- docker network rm teraflowbridge
		- docker volume prune --force
		- docker image prune --force
		@@ -143,13 +130,11 @@ unit_test analytics-frontend:
		- if docker container ls \| grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi
		- if docker volume ls \| grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi
		- if docker container ls \| grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi
		#- if docker container ls \| grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi
		- if docker container ls \| grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi
		- docker container prune -f
		script:
		- docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG"
		#- docker pull "bitnami/zookeeper:latest"
		- docker pull "bitnami/kafka:latest"
		- docker pull "bitnamilegacy/kafka:latest"
		- docker pull "cockroachdb/cockroach:latest-v22.2"
		- docker volume create crdb
		- >
		@@ -163,16 +148,6 @@ unit_test analytics-frontend:
		# - docker ps -a
		- CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
		- echo $CRDB_ADDRESS
		#- >
		# docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181
		# --env ALLOW_ANONYMOUS_LOGIN=yes
		# bitnami/zookeeper:latest
		#- sleep 10 # Wait for Zookeeper to start
		#- >
		# docker run --name kafka -d --network=teraflowbridge -p 9092:9092
		# --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
		# --env ALLOW_PLAINTEXT_LISTENER=yes
		# bitnami/kafka:latest
		- >
		docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093
		--env KAFKA_CFG_NODE_ID=1
		@@ -181,13 +156,12 @@ unit_test analytics-frontend:
		--env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
		--env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
		--env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
		bitnami/kafka:latest
		bitnamilegacy/kafka:latest
		- while ! docker logs kafka 2>&1 \| grep -q 'Kafka Server started'; do sleep 1; done
		- sleep 5 # Give extra time to Kafka to get stabilized
		- docker inspect kafka --format "{{.NetworkSettings.Networks}}"
		- KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
		- echo $KAFKA_IP
		#- docker logs zookeeper
		- docker logs kafka
		- >
		docker run --name $IMAGE_NAME-frontend -d -p 30050:30050
		@@ -207,7 +181,6 @@ unit_test analytics-frontend:
		after_script:
		- docker rm -f ${IMAGE_NAME}-frontend
		- docker rm -f crdb
		#- docker rm -f zookeeper
		- docker rm -f kafka
		- docker volume rm -f crdb
		- docker volume prune --force

src/automation/.gitlab-ci.yml

+81 −57

Original line number	Diff line number	Diff line
		@@ -39,56 +39,80 @@ build automation:
		- manifests/${IMAGE_NAME}service.yaml
		- .gitlab-ci.yml

		# Apply unit test to the component
		unit_test automation:
		variables:
		IMAGE_NAME: 'automation' # name of the microservice
		IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
		stage: unit_test
		needs:
		- build automation
		before_script:
		- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		- >
		if docker network list \| grep teraflowbridge; then
		echo "teraflowbridge is already created";
		else
		docker network create -d bridge teraflowbridge;
		fi
		- >
		if docker container ls \| grep $IMAGE_NAME; then
		docker rm -f $IMAGE_NAME;
		else
		echo "$IMAGE_NAME image is not in the system";
		fi
		script:
		- docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		- docker run --name $IMAGE_NAME -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		- sleep 5
		- docker ps -a
		- docker logs $IMAGE_NAME
		- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_automation_handlers.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml"
		- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
		coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
		after_script:
		- docker rm -f $IMAGE_NAME
		- docker network rm teraflowbridge
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		- changes:
		- src/common/*/.py
		- proto/*.proto
		- src/$IMAGE_NAME/*/.{py,in,yml}
		- src/$IMAGE_NAME/Dockerfile
		- src/$IMAGE_NAME/tests/*.py
		- src/$IMAGE_NAME/tests/Dockerfile
		- manifests/${IMAGE_NAME}service.yaml
		- .gitlab-ci.yml
		artifacts:
		when: always
		reports:
		junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
		# Deactivated as import `from ..service.zsm_handlers import Poc1 , Poc2` in `test_automation_handlers.py` crashes:
		## Apply unit test to the component
		#unit_test automation:
		# variables:
		# IMAGE_NAME: 'automation' # name of the microservice
		# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
		# stage: unit_test
		# needs:
		# - build automation
		# before_script:
		# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		# - >
		# if docker network list \| grep teraflowbridge; then
		# echo "teraflowbridge is already created";
		# else
		# docker network create -d bridge teraflowbridge;
		# fi
		# - >
		# if docker container ls \| grep $IMAGE_NAME; then
		# docker rm -f $IMAGE_NAME;
		# else
		# echo "$IMAGE_NAME image is not in the system";
		# fi
		# script:
		# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		# - docker pull "cockroachdb/cockroach:latest-v22.2"
		# - docker volume create crdb
		# - >
		# docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080
		# --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123
		# --volume "crdb:/cockroach/cockroach-data"
		# cockroachdb/cockroach:latest-v22.2 start-single-node
		# - echo "Waiting for initialization..."
		# - while ! docker logs crdb 2>&1 \| grep -q 'finished creating default user \"tfs\"'; do sleep 1; done
		# - docker logs crdb
		# - docker ps -a
		# - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
		# - echo $CRDB_ADDRESS
		# - >
		# docker run --name $IMAGE_NAME -d -p 2020:2020
		# --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require"
		# --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results"
		# --network=teraflowbridge
		# $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		# - docker ps -a
		# - sleep 5
		# - docker logs $IMAGE_NAME
		# - >
		# docker exec -i $IMAGE_NAME bash -c
		# "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py"
		# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
		# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
		# after_script:
		# - docker rm -f $IMAGE_NAME crdb
		# - docker volume rm -f crdb
		# - docker network rm teraflowbridge
		# - docker volume prune --force
		# - docker image prune --force
		# rules:
		# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		# - changes:
		# - src/common/*/.py
		# - proto/*.proto
		# - src/$IMAGE_NAME/*/.{py,in,yml}
		# - src/$IMAGE_NAME/Dockerfile
		# - src/$IMAGE_NAME/tests/*.py
		# - src/$IMAGE_NAME/tests/Dockerfile
		# - manifests/${IMAGE_NAME}service.yaml
		# - .gitlab-ci.yml
		# artifacts:
		# when: always
		# reports:
		# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml

		## Deployment of the service in Kubernetes Cluster
		#deploy automation:

src/automation/service/AutomationServiceServicerImpl.py

+22 −13

Original line number	Diff line number	Diff line
		@@ -12,20 +12,21 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import grpc, json, logging
		import grpc, logging
		from uuid import uuid4
		from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
		from common.proto.automation_pb2_grpc import AutomationServiceServicer
		from common.method_wrappers.ServiceExceptions import InvalidArgumentException
		from common.proto.automation_pb2 import ZSMCreateRequest, ZSMService, ZSMServiceID, ZSMServiceState, ZSMCreateUpdate
		from common.proto.context_pb2 import Service, ServiceId
		from automation.service.zsm_handlers import ZSM_SERVICE_HANDLERS
		from automation.service.zsm_handler_api.ZSMFilterFields import ZSMFilterFieldEnum , TELEMETRY_SERVICE_TYPE_VALUES, TARGET_SERVICE_TYPE_VALUES , ZSM_FILTER_FIELD_ALLOWED_VALUES
		from common.proto.context_pb2 import ServiceTypeEnum , DeviceDriverEnum
		from context.client.ContextClient import ContextClient
		from common.method_wrappers.ServiceExceptions import NotFoundException
		from common.proto.automation_pb2 import ZSMCreateRequest, ZSMService, ZSMServiceID, ZSMServiceState
		from common.proto.context_pb2 import ServiceId
		from automation.service.database.AutomationDB import AutomationDB
		from automation.service.database.AutomationModel import AutomationModel
		from common.method_wrappers.ServiceExceptions import NotFoundException
		from automation.service.zsm_handlers import ZSM_SERVICE_HANDLERS
		from automation.service.zsm_handler_api.ZSMFilterFields import (
		ZSMFilterFieldEnum, #TELEMETRY_SERVICE_TYPE_VALUES, TARGET_SERVICE_TYPE_VALUES,
		ZSM_FILTER_FIELD_ALLOWED_VALUES
		)
		from context.client.ContextClient import ContextClient

		LOGGER = logging.getLogger(__name__)
		METRICS_POOL = MetricsPool('Automation', 'RPC')
		@@ -43,7 +44,9 @@ class AutomationServiceServicerImpl(AutomationServiceServicer):
		targetService = context_client.GetService(request.target_service_id)
		telemetryService = context_client.GetService(request.telemetry_service_id)

		handler_cls = self.get_service_handler_based_on_service_types(targetService.service_type, telemetryService.service_type, ZSM_SERVICE_HANDLERS)
		handler_cls = self.get_service_handler_based_on_service_types(
		targetService.service_type, telemetryService.service_type, ZSM_SERVICE_HANDLERS
		)

		if handler_cls:
		handler_obj = handler_cls() # instantiate it
		@@ -93,16 +96,22 @@ class AutomationServiceServicerImpl(AutomationServiceServicer):
		LOGGER.info('NOT IMPLEMENTED ZSMGetByService')
		return ZSMService()

		def get_service_handler_based_on_service_types(self, targetServiceType ,telemetryServiceType , ZSM_SERVICE_HANDLERS):
		def get_service_handler_based_on_service_types(
		self, targetServiceType ,telemetryServiceType , ZSM_SERVICE_HANDLERS
		):
		flag = True
		for handler_cls, filters in ZSM_SERVICE_HANDLERS:
		for filter in filters:
		flag = self.check_if_requested_services_pass_filter_criteria(filter , targetServiceType, telemetryServiceType)
		flag = self.check_if_requested_services_pass_filter_criteria(
		filter, targetServiceType, telemetryServiceType
		)
		if flag:
		return handler_cls
		return None

		def check_if_requested_services_pass_filter_criteria(self ,filter , targetServiceType , telemetryServiceType):
		def check_if_requested_services_pass_filter_criteria(
		self ,filter , targetServiceType , telemetryServiceType
		):
		flag = True
		for filter_key, filter_value in filter.items():
		if filter_value in ZSM_FILTER_FIELD_ALLOWED_VALUES[filter_key.value]: