From 4393820c221923bb7401498b97df441ad27a2ab3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 8 Oct 2025 15:42:32 +0000 Subject: [PATCH 1/8] Kafka deployment artifacts: - Upgraded manifests - Upgraded scripts - Upgraded CI/CD pipeline descriptors --- manifests/kafka/single-node.yaml | 10 +------- scripts/run_tests_locally-nbi-all.sh | 4 +-- src/analytics/.gitlab-ci.yml | 37 ++++------------------------ src/kpi_value_api/.gitlab-ci.yml | 18 ++------------ src/kpi_value_writer/.gitlab-ci.yml | 18 ++------------ src/nbi/.gitlab-ci.yml | 4 +-- src/telemetry/.gitlab-ci.yml | 37 ++++------------------------ 7 files changed, 19 insertions(+), 109 deletions(-) diff --git a/manifests/kafka/single-node.yaml b/manifests/kafka/single-node.yaml index 4c435c11b..211f1858d 100644 --- a/manifests/kafka/single-node.yaml +++ b/manifests/kafka/single-node.yaml @@ -60,7 +60,7 @@ spec: restartPolicy: Always containers: - name: kafka - image: bitnami/kafka:latest + image: bitnamilegacy/kafka:latest imagePullPolicy: IfNotPresent ports: - name: clients @@ -68,14 +68,6 @@ spec: - name: control-plane containerPort: 9093 env: - #- name: KAFKA_BROKER_ID - # value: "1" - #- name: KAFKA_ZOOKEEPER_CONNECT - # value: zookeeper-service..svc.cluster.local:2181 - #- name: KAFKA_LISTENERS - # value: PLAINTEXT://:9092 - #- name: KAFKA_ADVERTISED_LISTENERS - # value: PLAINTEXT://kafka-service..svc.cluster.local:9092 - name: KAFKA_CFG_NODE_ID value: "1" - name: KAFKA_CFG_PROCESS_ROLES diff --git a/scripts/run_tests_locally-nbi-all.sh b/scripts/run_tests_locally-nbi-all.sh index 30a5ccd13..35c755b38 100755 --- a/scripts/run_tests_locally-nbi-all.sh +++ b/scripts/run_tests_locally-nbi-all.sh @@ -18,7 +18,7 @@ docker ps -aq | xargs -r docker rm -f docker network rm teraflowbridge || true docker container prune -f -docker pull "bitnami/kafka:latest" +docker pull "bitnamilegacy/kafka:latest" docker buildx build -t "mock_tfs_nbi_dependencies:test" -f ./src/tests/tools/mock_tfs_nbi_dependencies/Dockerfile . docker buildx build -t "nbi:latest" -f ./src/nbi/Dockerfile . docker image prune --force @@ -32,7 +32,7 @@ docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 \ --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT \ --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \ --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 \ - bitnami/kafka:latest + bitnamilegacy/kafka:latest echo "Waiting for Kafka to be ready..." while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do diff --git a/src/analytics/.gitlab-ci.yml b/src/analytics/.gitlab-ci.yml index f5bf4abb4..6f48b9db1 100644 --- a/src/analytics/.gitlab-ci.yml +++ b/src/analytics/.gitlab-ci.yml @@ -60,24 +60,12 @@ unit_test analytics-backend: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - # - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest + - docker pull "bitnamilegacy/kafka:latest" - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -86,7 +74,7 @@ unit_test analytics-backend: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" @@ -109,7 +97,6 @@ unit_test analytics-backend: after_script: - docker rm -f ${IMAGE_NAME}-backend - docker rm -f kafka - #- docker rm -f zookeeper - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force @@ -143,13 +130,11 @@ unit_test analytics-frontend: - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" + - docker pull "bitnamilegacy/kafka:latest" - docker pull "cockroachdb/cockroach:latest-v22.2" - docker volume create crdb - > @@ -163,16 +148,6 @@ unit_test analytics-frontend: # - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -181,13 +156,12 @@ unit_test analytics-frontend: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - #- docker logs zookeeper - docker logs kafka - > docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 @@ -207,7 +181,6 @@ unit_test analytics-frontend: after_script: - docker rm -f ${IMAGE_NAME}-frontend - docker rm -f crdb - #- docker rm -f zookeeper - docker rm -f kafka - docker volume rm -f crdb - docker volume prune --force diff --git a/src/kpi_value_api/.gitlab-ci.yml b/src/kpi_value_api/.gitlab-ci.yml index 146bae380..eafc285c8 100644 --- a/src/kpi_value_api/.gitlab-ci.yml +++ b/src/kpi_value_api/.gitlab-ci.yml @@ -52,22 +52,10 @@ unit_test kpi-value-api: - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest + - docker pull "bitnamilegacy/kafka:latest" - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -76,7 +64,7 @@ unit_test kpi-value-api: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" @@ -90,7 +78,6 @@ unit_test kpi-value-api: $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - #- docker logs zookeeper - docker logs kafka - docker logs $IMAGE_NAME - > @@ -102,7 +89,6 @@ unit_test kpi-value-api: - docker ps -a - docker rm -f $IMAGE_NAME - docker rm -f kafka - #- docker rm -f zookeeper - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index ea08a658a..d681b9f9e 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -52,22 +52,10 @@ unit_test kpi-value-writer: - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest + - docker pull "bitnamilegacy/kafka:latest" - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -76,7 +64,7 @@ unit_test kpi-value-writer: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" @@ -90,7 +78,6 @@ unit_test kpi-value-writer: $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - #- docker logs zookeeper - docker logs kafka - docker logs $IMAGE_NAME - > @@ -102,7 +89,6 @@ unit_test kpi-value-writer: - docker ps -a - docker rm -f $IMAGE_NAME - docker rm -f kafka - #- docker rm -f zookeeper - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/nbi/.gitlab-ci.yml b/src/nbi/.gitlab-ci.yml index 9096a9214..1a4bfe01b 100644 --- a/src/nbi/.gitlab-ci.yml +++ b/src/nbi/.gitlab-ci.yml @@ -73,7 +73,7 @@ unit_test nbi: script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/mock_tfs_nbi_dependencies:test" - - docker pull "bitnami/kafka:latest" + - docker pull "bitnamilegacy/kafka:latest" - docker image prune --force - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 @@ -83,7 +83,7 @@ unit_test nbi: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 54e6c095e..8392652da 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -60,24 +60,12 @@ unit_test telemetry-backend: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - # - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest + - docker pull "bitnamilegacy/kafka:latest" - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -86,7 +74,7 @@ unit_test telemetry-backend: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" @@ -109,7 +97,6 @@ unit_test telemetry-backend: after_script: - docker rm -f ${IMAGE_NAME}-backend - docker rm -f kafka - #- docker rm -f zookeeper - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force @@ -143,13 +130,11 @@ unit_test telemetry-frontend: - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi - #- if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" - #- docker pull "bitnami/zookeeper:latest" - - docker pull "bitnami/kafka:latest" + - docker pull "bitnamilegacy/kafka:latest" - docker pull "cockroachdb/cockroach:latest-v22.2" - docker volume create crdb - > @@ -163,16 +148,6 @@ unit_test telemetry-frontend: # - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - #- > - # docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - # --env ALLOW_ANONYMOUS_LOGIN=yes - # bitnami/zookeeper:latest - #- sleep 10 # Wait for Zookeeper to start - #- > - # docker run --name kafka -d --network=teraflowbridge -p 9092:9092 - # --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # --env ALLOW_PLAINTEXT_LISTENER=yes - # bitnami/kafka:latest - > docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -p 9093:9093 --env KAFKA_CFG_NODE_ID=1 @@ -181,13 +156,12 @@ unit_test telemetry-frontend: --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 - bitnami/kafka:latest + bitnamilegacy/kafka:latest - while ! docker logs kafka 2>&1 | grep -q 'Kafka Server started'; do sleep 1; done - sleep 5 # Give extra time to Kafka to get stabilized - docker inspect kafka --format "{{.NetworkSettings.Networks}}" - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - #- docker logs zookeeper - docker logs kafka - > docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 @@ -207,7 +181,6 @@ unit_test telemetry-frontend: after_script: - docker rm -f ${IMAGE_NAME}-frontend - docker rm -f crdb - #- docker rm -f zookeeper - docker rm -f kafka - docker volume rm -f crdb - docker volume prune --force -- GitLab From 8cf37415e41ea2a1f98964c35c4f1a89c6767f10 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 8 Oct 2025 15:55:46 +0000 Subject: [PATCH 2/8] Automation component: - Resolved import issue - Minor code formatting --- .../service/AutomationServiceServicerImpl.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/automation/service/AutomationServiceServicerImpl.py b/src/automation/service/AutomationServiceServicerImpl.py index 984c0f1f6..187eba055 100644 --- a/src/automation/service/AutomationServiceServicerImpl.py +++ b/src/automation/service/AutomationServiceServicerImpl.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc, json, logging +import grpc, logging from uuid import uuid4 from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.automation_pb2_grpc import AutomationServiceServicer -from common.method_wrappers.ServiceExceptions import InvalidArgumentException -from common.proto.automation_pb2 import ZSMCreateRequest, ZSMService, ZSMServiceID, ZSMServiceState, ZSMCreateUpdate -from common.proto.context_pb2 import Service, ServiceId -from automation.service.zsm_handlers import ZSM_SERVICE_HANDLERS -from automation.service.zsm_handler_api.ZSMFilterFields import ZSMFilterFieldEnum , TELEMETRY_SERVICE_TYPE_VALUES, TARGET_SERVICE_TYPE_VALUES , ZSM_FILTER_FIELD_ALLOWED_VALUES -from common.proto.context_pb2 import ServiceTypeEnum , DeviceDriverEnum -from context.client.ContextClient import ContextClient +from common.method_wrappers.ServiceExceptions import NotFoundException +from common.proto.automation_pb2 import ZSMCreateRequest, ZSMService, ZSMServiceID, ZSMServiceState +from common.proto.context_pb2 import ServiceId from automation.service.database.AutomationDB import AutomationDB from automation.service.database.AutomationModel import AutomationModel -from common.method_wrappers.ServiceExceptions import NotFoundException +from automation.service.zsm_handlers import ZSM_SERVICE_HANDLERS +from automation.service.zsm_handler_api.ZSMFilterFields import ( + ZSMFilterFieldEnum, #TELEMETRY_SERVICE_TYPE_VALUES, TARGET_SERVICE_TYPE_VALUES, + ZSM_FILTER_FIELD_ALLOWED_VALUES +) +from context.client.ContextClient import ContextClient LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('Automation', 'RPC') @@ -43,7 +44,9 @@ class AutomationServiceServicerImpl(AutomationServiceServicer): targetService = context_client.GetService(request.target_service_id) telemetryService = context_client.GetService(request.telemetry_service_id) - handler_cls = self.get_service_handler_based_on_service_types(targetService.service_type, telemetryService.service_type, ZSM_SERVICE_HANDLERS) + handler_cls = self.get_service_handler_based_on_service_types( + targetService.service_type, telemetryService.service_type, ZSM_SERVICE_HANDLERS + ) if handler_cls: handler_obj = handler_cls() # instantiate it @@ -93,16 +96,22 @@ class AutomationServiceServicerImpl(AutomationServiceServicer): LOGGER.info('NOT IMPLEMENTED ZSMGetByService') return ZSMService() - def get_service_handler_based_on_service_types(self, targetServiceType ,telemetryServiceType , ZSM_SERVICE_HANDLERS): + def get_service_handler_based_on_service_types( + self, targetServiceType ,telemetryServiceType , ZSM_SERVICE_HANDLERS + ): flag = True for handler_cls, filters in ZSM_SERVICE_HANDLERS: for filter in filters: - flag = self.check_if_requested_services_pass_filter_criteria(filter , targetServiceType, telemetryServiceType) + flag = self.check_if_requested_services_pass_filter_criteria( + filter, targetServiceType, telemetryServiceType + ) if flag: return handler_cls return None - def check_if_requested_services_pass_filter_criteria(self ,filter , targetServiceType , telemetryServiceType): + def check_if_requested_services_pass_filter_criteria( + self ,filter , targetServiceType , telemetryServiceType + ): flag = True for filter_key, filter_value in filter.items(): if filter_value in ZSM_FILTER_FIELD_ALLOWED_VALUES[filter_key.value]: -- GitLab From 6e027915d8a1082b662ac1a89458caa19fe0936e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 8 Oct 2025 15:58:02 +0000 Subject: [PATCH 3/8] Automation component: - Resolved import issue - Minor code formatting --- src/automation/service/zsm_handler_api/_ZSMHandler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/automation/service/zsm_handler_api/_ZSMHandler.py b/src/automation/service/zsm_handler_api/_ZSMHandler.py index 01d2614e2..e79601cf7 100644 --- a/src/automation/service/zsm_handler_api/_ZSMHandler.py +++ b/src/automation/service/zsm_handler_api/_ZSMHandler.py @@ -13,7 +13,10 @@ # limitations under the License. import grpc , logging -from common.proto.automation_pb2 import ZSMCreateRequest, ZSMService, ZSMServiceID, ZSMServiceState, ZSMCreateUpdate +from common.proto.automation_pb2 import ( + ZSMCreateRequest, #ZSMService, + ZSMServiceID, #ZSMServiceState, +) from common.proto.context_pb2 import ServiceId LOGGER = logging.getLogger(__name__) -- GitLab From 9492cf6715b434ddcb13c48aaa3fa904d001db93 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 8 Oct 2025 16:06:41 +0000 Subject: [PATCH 4/8] CI/CD pipelines: - Fixed piepline for Automation component - Minor code improvements --- src/automation/.gitlab-ci.yml | 61 ++++++++++++++++++++++++----------- src/context/.gitlab-ci.yml | 2 +- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/automation/.gitlab-ci.yml b/src/automation/.gitlab-ci.yml index 584656248..7fadafcc8 100644 --- a/src/automation/.gitlab-ci.yml +++ b/src/automation/.gitlab-ci.yml @@ -31,13 +31,13 @@ build automation: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml # Apply unit test to the component unit_test automation: @@ -63,28 +63,51 @@ unit_test automation: fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - sleep 5 + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - > + docker run --name $IMAGE_NAME -d -p 2020:2020 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - docker ps -a + - sleep 5 - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_automation_handlers.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - - docker rm -f $IMAGE_NAME + - docker rm -f $IMAGE_NAME crdb + - docker volume rm -f crdb - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml artifacts: when: always reports: diff --git a/src/context/.gitlab-ci.yml b/src/context/.gitlab-ci.yml index 0abebbd3c..342595dff 100644 --- a/src/context/.gitlab-ci.yml +++ b/src/context/.gitlab-ci.yml @@ -139,6 +139,6 @@ unit_test context: # # namespace: test # rules: # - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# when: manual +# when: manual # - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' # when: manual -- GitLab From 049c77037ca9f88685adc4d800ac193cd599b9e2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 8 Oct 2025 17:37:49 +0000 Subject: [PATCH 5/8] Automation component: - Disable incomplete CI/CD pipeline --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca970101f..d68469698 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ include: - local: '/src/opticalcontroller/.gitlab-ci.yml' - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' - - local: '/src/automation/.gitlab-ci.yml' + #- local: '/src/automation/.gitlab-ci.yml' - local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' -- GitLab From 2c9654b5b31ec150451a5d92264ce5cacd951a26 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 9 Oct 2025 06:54:52 +0000 Subject: [PATCH 6/8] Automation component: - Enable basic CI/CD pipeline --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d68469698..ca970101f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ include: - local: '/src/opticalcontroller/.gitlab-ci.yml' - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' - #- local: '/src/automation/.gitlab-ci.yml' + - local: '/src/automation/.gitlab-ci.yml' - local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' -- GitLab From a7b802ae5fd49cfa6cd9e1ae44284a6581cd4c03 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 9 Oct 2025 07:01:29 +0000 Subject: [PATCH 7/8] Automation component: - Disabled wrong test in CI/CD pipeline --- src/automation/.gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/automation/.gitlab-ci.yml b/src/automation/.gitlab-ci.yml index 7fadafcc8..9e813c92e 100644 --- a/src/automation/.gitlab-ci.yml +++ b/src/automation/.gitlab-ci.yml @@ -85,9 +85,11 @@ unit_test automation: - docker ps -a - sleep 5 - docker logs $IMAGE_NAME - - > - docker exec -i $IMAGE_NAME bash -c - "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py" + + # Deactivated as import `from ..service.zsm_handlers import Poc1 , Poc2` in `test_automation_handlers.py` crashes: + #- > + # docker exec -i $IMAGE_NAME bash -c + # "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: -- GitLab From 129afbd31e0b2afc2e0dec147834a185bdd31c2f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 9 Oct 2025 07:37:34 +0000 Subject: [PATCH 8/8] Automation component: - Disabled wrong test in CI/CD pipeline --- src/automation/.gitlab-ci.yml | 149 +++++++++++++++++----------------- 1 file changed, 74 insertions(+), 75 deletions(-) diff --git a/src/automation/.gitlab-ci.yml b/src/automation/.gitlab-ci.yml index 9e813c92e..568292717 100644 --- a/src/automation/.gitlab-ci.yml +++ b/src/automation/.gitlab-ci.yml @@ -39,81 +39,80 @@ build automation: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# Apply unit test to the component -unit_test automation: - variables: - IMAGE_NAME: 'automation' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build automation - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - > - if docker network list | grep teraflowbridge; then - echo "teraflowbridge is already created"; - else - docker network create -d bridge teraflowbridge; - fi - - > - if docker container ls | grep $IMAGE_NAME; then - docker rm -f $IMAGE_NAME; - else - echo "$IMAGE_NAME image is not in the system"; - fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker pull "cockroachdb/cockroach:latest-v22.2" - - docker volume create crdb - - > - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 - --volume "crdb:/cockroach/cockroach-data" - cockroachdb/cockroach:latest-v22.2 start-single-node - - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - - docker logs crdb - - docker ps -a - - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CRDB_ADDRESS - - > - docker run --name $IMAGE_NAME -d -p 2020:2020 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - docker ps -a - - sleep 5 - - docker logs $IMAGE_NAME - - # Deactivated as import `from ..service.zsm_handlers import Poc1 , Poc2` in `test_automation_handlers.py` crashes: - #- > - # docker exec -i $IMAGE_NAME bash -c - # "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME crdb - - docker volume rm -f crdb - - docker network rm teraflowbridge - - docker volume prune --force - - docker image prune --force - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +# Deactivated as import `from ..service.zsm_handlers import Poc1 , Poc2` in `test_automation_handlers.py` crashes: +## Apply unit test to the component +#unit_test automation: +# variables: +# IMAGE_NAME: 'automation' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build automation +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - > +# if docker network list | grep teraflowbridge; then +# echo "teraflowbridge is already created"; +# else +# docker network create -d bridge teraflowbridge; +# fi +# - > +# if docker container ls | grep $IMAGE_NAME; then +# docker rm -f $IMAGE_NAME; +# else +# echo "$IMAGE_NAME image is not in the system"; +# fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker pull "cockroachdb/cockroach:latest-v22.2" +# - docker volume create crdb +# - > +# docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 +# --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 +# --volume "crdb:/cockroach/cockroach-data" +# cockroachdb/cockroach:latest-v22.2 start-single-node +# - echo "Waiting for initialization..." +# - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done +# - docker logs crdb +# - docker ps -a +# - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") +# - echo $CRDB_ADDRESS +# - > +# docker run --name $IMAGE_NAME -d -p 2020:2020 +# --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" +# --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" +# --network=teraflowbridge +# $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - docker ps -a +# - sleep 5 +# - docker logs $IMAGE_NAME +# - > +# docker exec -i $IMAGE_NAME bash -c +# "coverage run --append -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_automation_handlers.py" +# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME crdb +# - docker volume rm -f crdb +# - docker network rm teraflowbridge +# - docker volume prune --force +# - docker image prune --force +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy automation: -- GitLab