From acf05e39ac3b46b60c8ff14c906d07ff4e44e822 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 1 Oct 2024 11:14:51 +0000 Subject: [PATCH 1/5] Fixed manifests and CI/CD for KPI manager, Telemetry, and Analytics --- manifests/kpi_managerservice.yaml | 2 +- manifests/telemetryservice.yaml | 2 +- src/analytics/.gitlab-ci.yml | 204 ++++++++++++++++++++++++++++++ src/telemetry/.gitlab-ci.yml | 5 +- 4 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 src/analytics/.gitlab-ci.yml diff --git a/manifests/kpi_managerservice.yaml b/manifests/kpi_managerservice.yaml index 31eaf1284..f8ee8ff82 100644 --- a/manifests/kpi_managerservice.yaml +++ b/manifests/kpi_managerservice.yaml @@ -40,7 +40,7 @@ spec: - name: LOG_LEVEL value: "INFO" - name: CRDB_DATABASE - value: "tfs_kpi" + value: "tfs_kpi_mgmt" envFrom: - secretRef: name: crdb-data diff --git a/manifests/telemetryservice.yaml b/manifests/telemetryservice.yaml index c3763d6a9..cd35d2698 100644 --- a/manifests/telemetryservice.yaml +++ b/manifests/telemetryservice.yaml @@ -38,7 +38,7 @@ spec: - name: LOG_LEVEL value: "INFO" - name: CRDB_DATABASE - value: "tfs_kpi" + value: "tfs_telemetry" - name: METRICS_PORT value: "9192" envFrom: diff --git a/src/analytics/.gitlab-ci.yml b/src/analytics/.gitlab-ci.yml new file mode 100644 index 000000000..6992443d7 --- /dev/null +++ b/src/analytics/.gitlab-ci.yml @@ -0,0 +1,204 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build analytics: + variables: + IMAGE_NAME: 'analytics' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + # This first build tags the builder resulting image to prevent being removed by dangling image removal command + # - docker buildx build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . + - docker tag "${IMAGE_NAME}-frontend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker tag "${IMAGE_NAME}-backend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/.gitlab-ci.yml + - src/$IMAGE_NAME/frontend/**/*.{py,in,yml} + - src/$IMAGE_NAME/frontend/Dockerfile + - src/$IMAGE_NAME/frontend/tests/*.py + - src/$IMAGE_NAME/backend/Dockerfile + - src/$IMAGE_NAME/backend/**/*.{py,in,yml} + - src/$IMAGE_NAME/backend/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test analytics-backend: + variables: + IMAGE_NAME: 'analytics' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build analytics + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi + # - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" + - > + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - > + docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + - > + docker run --name $IMAGE_NAME-backend -d -p 30060:30060 + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/backend/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs ${IMAGE_NAME}-backend + - > + docker exec -i ${IMAGE_NAME}-backend bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}-backend_report.xml $IMAGE_NAME/backend/tests/test_*.py" + - docker exec -i ${IMAGE_NAME}-backend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + - docker rm -f ${IMAGE_NAME}-backend + - docker rm -f zookeeper + - docker rm -f kafka + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/backend/**/*.{py,in,yml} + - src/$IMAGE_NAME/backend/Dockerfile + - src/$IMAGE_NAME/backend/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/backend/tests/${IMAGE_NAME}-backend_report.xml + +# Apply unit test to the component +unit_test analytics-frontend: + variables: + IMAGE_NAME: 'analytics' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build analytics + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + # - docker logs crdb + # - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - > + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 \ + -e ALLOW_ANONYMOUS_LOGIN=yes \ + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + # - docker logs zookeeper + # - docker logs kafka + - > + docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/frontend/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs ${IMAGE_NAME}-frontend + - > + docker exec -i ${IMAGE_NAME}-frontend bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}-frontend_report.xml $IMAGE_NAME/frontend/tests/test_*.py" + - docker exec -i ${IMAGE_NAME}-frontend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + - docker rm -f ${IMAGE_NAME}-frontend + - docker rm -f zookeeper + - docker rm -f kafka + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/frontend/**/*.{py,in,yml} + - src/$IMAGE_NAME/frontend/Dockerfile + - src/$IMAGE_NAME/frontend/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/frontend/tests/${IMAGE_NAME}-frontend_report.xml diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 110a6490d..9375c339e 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -71,7 +71,8 @@ unit_test telemetry-backend: docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + - > + docker run --name kafka -d --network=teraflowbridge -p 9092:9092 --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env ALLOW_PLAINTEXT_LISTENER=yes bitnami/kafka:latest @@ -200,4 +201,4 @@ unit_test telemetry-frontend: artifacts: when: always reports: - junit: src/$IMAGE_NAME/frontend/tests/${IMAGE_NAME}-frontend_report.xml \ No newline at end of file + junit: src/$IMAGE_NAME/frontend/tests/${IMAGE_NAME}-frontend_report.xml -- GitLab From 637c484028f9cd749c998a505627291f93356964 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 1 Oct 2024 12:33:52 +0000 Subject: [PATCH 2/5] Fix Telemetry and KPI Manager CI/CD pipeline --- src/kpi_manager/tests/test_kpi_db.py | 9 ++++----- src/telemetry/.gitlab-ci.yml | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/kpi_manager/tests/test_kpi_db.py b/src/kpi_manager/tests/test_kpi_db.py index b1513a83f..44c6df6a9 100644 --- a/src/kpi_manager/tests/test_kpi_db.py +++ b/src/kpi_manager/tests/test_kpi_db.py @@ -14,18 +14,17 @@ import logging -# from kpi_manager.database.Kpi_DB import KpiDB -from common.proto.kpi_manager_pb2 import KpiDescriptorList -from .test_messages import create_kpi_filter_request -from kpi_manager.database.KpiModel import Kpi as KpiModel +#from common.proto.kpi_manager_pb2 import KpiDescriptorList +#from .test_messages import create_kpi_filter_request from kpi_manager.database.KpiDB import KpiDB +from kpi_manager.database.KpiModel import Kpi as KpiModel # from common.tools.database.GenericDatabase import Database LOGGER = logging.getLogger(__name__) def test_verify_databases_and_Tables(): LOGGER.info('>>> test_verify_Tables : START <<< ') - kpiDBobj = KpiDB() + kpiDBobj = KpiDB(KpiModel) # kpiDBobj.drop_database() # kpiDBobj.verify_tables() kpiDBobj.create_database() diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 9375c339e..11ee4bb6a 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -151,8 +151,8 @@ unit_test telemetry-frontend: - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - > - docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 \ - -e ALLOW_ANONYMOUS_LOGIN=yes \ + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 -- GitLab From d28cfb9846f75bab4812b300ac4d00b424637e12 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 1 Oct 2024 15:09:19 +0000 Subject: [PATCH 3/5] Fix Telemetry Analytics, KPI Manager CI/CD pipeline --- .gitlab-ci.yml | 1 + src/analytics/.gitlab-ci.yml | 7 ++++--- src/kpi_value_writer/.gitlab-ci.yml | 5 ++++- src/telemetry/.gitlab-ci.yml | 3 ++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb6ea273b..a67d2346c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,5 +50,6 @@ include: - local: '/src/kpi_value_writer/.gitlab-ci.yml' - local: '/src/telemetry/.gitlab-ci.yml' - local: '/src/analytics/.gitlab-ci.yml' + # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/analytics/.gitlab-ci.yml b/src/analytics/.gitlab-ci.yml index 6992443d7..8f17fbfdb 100644 --- a/src/analytics/.gitlab-ci.yml +++ b/src/analytics/.gitlab-ci.yml @@ -151,11 +151,12 @@ unit_test analytics-frontend: - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - > - docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 \ - -e ALLOW_ANONYMOUS_LOGIN=yes \ + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + - > + docker run --name kafka -d --network=teraflowbridge -p 9092:9092 --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env ALLOW_PLAINTEXT_LISTENER=yes bitnami/kafka:latest diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index 9a2f9fd47..f00b9208d 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -61,7 +61,8 @@ unit_test kpi-value-writer: docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + - > + docker run --name kafka -d --network=teraflowbridge -p 9092:9092 --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env ALLOW_PLAINTEXT_LISTENER=yes bitnami/kafka:latest @@ -76,6 +77,8 @@ unit_test kpi-value-writer: $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a + - docker logs zookeeper + - docker logs kafka - docker logs $IMAGE_NAME - > docker exec -i $IMAGE_NAME bash -c diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 11ee4bb6a..68ffbda52 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -155,7 +155,8 @@ unit_test telemetry-frontend: -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + - > + docker run --name kafka -d --network=teraflowbridge -p 9092:9092 --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env ALLOW_PLAINTEXT_LISTENER=yes bitnami/kafka:latest -- GitLab From 8daa566cb496850a2210b64ee7349fea8a7d3575 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 1 Oct 2024 15:50:55 +0000 Subject: [PATCH 4/5] Fix Telemetry, Analytics, and KPI Manager CI/CD pipeline --- src/analytics/.gitlab-ci.yml | 21 +++++++++++---------- src/kpi_value_api/.gitlab-ci.yml | 3 +++ src/kpi_value_writer/.gitlab-ci.yml | 2 +- src/telemetry/.gitlab-ci.yml | 21 +++++++++++---------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/analytics/.gitlab-ci.yml b/src/analytics/.gitlab-ci.yml index 8f17fbfdb..dfc5f2195 100644 --- a/src/analytics/.gitlab-ci.yml +++ b/src/analytics/.gitlab-ci.yml @@ -69,6 +69,7 @@ unit_test analytics-backend: - docker pull "bitnami/kafka:latest" - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - > @@ -94,12 +95,12 @@ unit_test analytics-backend: - docker exec -i ${IMAGE_NAME}-backend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: + - docker rm -f ${IMAGE_NAME}-backend + - docker rm -f kafka + - docker rm -f zookeeper - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force - - docker rm -f ${IMAGE_NAME}-backend - - docker rm -f zookeeper - - docker rm -f kafka rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' @@ -152,7 +153,7 @@ unit_test analytics-frontend: - echo $CRDB_ADDRESS - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - -e ALLOW_ANONYMOUS_LOGIN=yes + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - > @@ -163,8 +164,8 @@ unit_test analytics-frontend: - sleep 20 # Wait for Kafka to start - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - # - docker logs zookeeper - # - docker logs kafka + - docker logs zookeeper + - docker logs kafka - > docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" @@ -181,13 +182,13 @@ unit_test analytics-frontend: - docker exec -i ${IMAGE_NAME}-frontend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - - docker volume rm -f crdb - - docker network rm teraflowbridge - - docker volume prune --force - - docker image prune --force - docker rm -f ${IMAGE_NAME}-frontend - docker rm -f zookeeper - docker rm -f kafka + - docker volume rm -f crdb + - docker volume prune --force + - docker image prune --force + - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' diff --git a/src/kpi_value_api/.gitlab-ci.yml b/src/kpi_value_api/.gitlab-ci.yml index 1a6f821ba..3db0538dd 100644 --- a/src/kpi_value_api/.gitlab-ci.yml +++ b/src/kpi_value_api/.gitlab-ci.yml @@ -59,6 +59,7 @@ unit_test kpi-value-api: - docker pull "bitnami/kafka:latest" - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 @@ -84,6 +85,8 @@ unit_test kpi-value-api: coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME + - docker rm -f kafka + - docker rm -f zookeeper - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index f00b9208d..065dbcfe7 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -87,8 +87,8 @@ unit_test kpi-value-writer: coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME - - docker rm -f zookeeper - docker rm -f kafka + - docker rm -f zookeeper - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 68ffbda52..358a93af8 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -69,6 +69,7 @@ unit_test telemetry-backend: - docker pull "bitnami/kafka:latest" - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - > @@ -94,12 +95,12 @@ unit_test telemetry-backend: - docker exec -i ${IMAGE_NAME}-backend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: + - docker rm -f ${IMAGE_NAME}-backend + - docker rm -f kafka + - docker rm -f zookeeper - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force - - docker rm -f ${IMAGE_NAME}-backend - - docker rm -f zookeeper - - docker rm -f kafka rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' @@ -152,7 +153,7 @@ unit_test telemetry-frontend: - echo $CRDB_ADDRESS - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 - -e ALLOW_ANONYMOUS_LOGIN=yes + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - > @@ -163,8 +164,8 @@ unit_test telemetry-frontend: - sleep 20 # Wait for Kafka to start - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - # - docker logs zookeeper - # - docker logs kafka + - docker logs zookeeper + - docker logs kafka - > docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" @@ -181,13 +182,13 @@ unit_test telemetry-frontend: - docker exec -i ${IMAGE_NAME}-frontend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - - docker volume rm -f crdb - - docker network rm teraflowbridge - - docker volume prune --force - - docker image prune --force - docker rm -f ${IMAGE_NAME}-frontend - docker rm -f zookeeper - docker rm -f kafka + - docker volume rm -f crdb + - docker volume prune --force + - docker image prune --force + - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -- GitLab From caea06226eabf4c4b548df0e41ea92ee22d32bf1 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 1 Oct 2024 16:22:06 +0000 Subject: [PATCH 5/5] Fix KPI Manager CI/CD pipeline --- src/kpi_value_writer/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index 065dbcfe7..3f376a673 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -59,6 +59,7 @@ unit_test kpi-value-writer: - docker pull "bitnami/kafka:latest" - > docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + --env ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - > -- GitLab