From 5db8c30d3506e3ca37dde6390224e52f50cc4e80 Mon Sep 17 00:00:00 2001 From: Sergio Date: Thu, 28 Oct 2021 13:51:35 +0200 Subject: [PATCH] Test monitoring component --- manifests/monitoring.yaml | 18 +- src/monitoring/.gitlab-ci.yml | 256 +++++++++--------- src/monitoring/Dockerfile | 12 - .../service/MonitoringServiceServicerImpl.py | 5 + src/monitoring/tests/Dockerfile | 14 - 5 files changed, 147 insertions(+), 158 deletions(-) diff --git a/manifests/monitoring.yaml b/manifests/monitoring.yaml index fc53381e7..15885b547 100644 --- a/manifests/monitoring.yaml +++ b/manifests/monitoring.yaml @@ -30,12 +30,12 @@ spec: envFrom: - secretRef: name: influxdb-secrets - # readinessProbe: - # exec: - # command: ["curl", "-XGET", "localhost:8086/health"] - # livenessProbe: - # exec: - # command: ["curl", "-XGET", "localhost:8086/health"] + readinessProbe: + exec: + command: ["curl", "-XGET", "localhost:8086/health"] + livenessProbe: + exec: + command: ["curl", "-XGET", "localhost:8086/health"] resources: requests: cpu: 250m @@ -48,9 +48,9 @@ spec: imagePullPolicy: Always ports: - containerPort: 7070 - env: - - name: PORT - value: "7070" + envFrom: + - secretRef: + name: monitoring-secrets readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:7070"] diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index 7cfd82bf1..1b47e831a 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -7,7 +7,7 @@ build monitoring: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build --build-arg hostname=localhost --build-arg user=$INFLUXDB_USER --build-arg pass=$INFLUXDB_PASSWORD --build-arg db=$INFLUXDB_DATABASE -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ + - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: @@ -23,56 +23,67 @@ build monitoring: - manifests/$IMAGE_NAME.yaml - .gitlab-ci.yml -# test if the Docker image can be pulled from the gitlab registry -test monitoring pull: - variables: - IMAGE_NAME: 'monitoring' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build monitoring - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +test kubernetes: + stage: test script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/$IMAGE_NAME.yaml - - .gitlab-ci.yml + - kubectl create secret generic monitoring-secrets --from-literal=INFLUXDB_DATABASE="monitoring" --from-literal=INFLUXDB_USER="teraflow" --from-literal=INFLUXDB_PASSWORD="teraflow" --from-literal=INFLUXDB_HOSTNAME="localhost" + - kubectl describe secret monitoring-secrets + - kubectl get all + - kubectl apply -f "manifests/monitoring.yaml" + - kubectl get all + # - kubectl --namespace gitlab-ci logs monitoring-5f946cdc5d-99sjp influxdb + # - kubectl --namespace gitlab-ci logs monitoring-5f946cdc5d-99sjp server -# test if the Docker image can be executed -test monitoring run: - variables: - IMAGE_NAME: 'monitoring' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build monitoring - before_script: - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker ps | grep monitoring; then docker stop monitoring; else echo "monitoring is not running"; fi - script: - - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker ps - after_script: - - docker stop "$IMAGE_NAME" - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/$IMAGE_NAME.yaml - - .gitlab-ci.yml +# # test if the Docker image can be pulled from the gitlab registry +# test monitoring pull: +# variables: +# IMAGE_NAME: 'monitoring' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build monitoring +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/$IMAGE_NAME.yaml +# - .gitlab-ci.yml + +# # test if the Docker image can be executed +# test monitoring run: +# variables: +# IMAGE_NAME: 'monitoring' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build monitoring +# before_script: +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker ps | grep monitoring; then docker stop monitoring; else echo "monitoring is not running"; fi +# script: +# - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker ps +# after_script: +# - docker stop "$IMAGE_NAME" +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/$IMAGE_NAME.yaml +# - .gitlab-ci.yml # apply unit test to the monitoring component test monitoring pytest: @@ -88,9 +99,8 @@ test monitoring pytest: - if docker container ls | grep influxdb; then docker rm influxdb; fi script: - docker run --name influxdb -d -p 8086:8086 -e INFLUXDB_DB=$INFLUXDB_DATABASE -e INFLUXDB_ADMIN_USER=$INFLUXDB_USER -e INFLUXDB_ADMIN_PASSWORD=$INFLUXDB_PASSWORD -e INXLUXDB_HTTP_AUTH_ENABLED=True --network=teraflowbridge influxdb:1.8 - # - docker build --build-arg hostname=influxdb --build-arg user=$INFLUXDB_USER --build-arg pass=$INFLUXDB_PASSWORD --build-arg db=$INFLUXDB_DATABASE -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/ - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/ - - docker run --name $IMAGE_NAME_TEST --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $IMAGE_NAME_TEST:$IMAGE_TAG + - docker run --name $IMAGE_NAME_TEST -p 7070:7070 --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $IMAGE_NAME_TEST:$IMAGE_TAG after_script: - docker stop influxdb - docker rm influxdb @@ -111,77 +121,77 @@ test monitoring pytest: reports: junit: src/$IMAGE_NAME/tests/report.xml -# Deployment of the monitoring service in testing Kubernetes Cluster -deploy monitoring test: - variables: - IMAGE_NAME: 'monitoring' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: deploy - needs: - - test monitoring pull - - test monitoring run - - test monitoring pytest - script: - - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' - - kubectl version - - kubectl get all - - kubectl apply -f "manifests/$IMAGE_NAME.yaml" - - kubectl get all - # environment: - # name: test - # url: https://example.com - # kubernetes: - # namespace: test - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"' - when: manual +# # Deployment of the monitoring service in testing Kubernetes Cluster +# deploy monitoring test: +# variables: +# IMAGE_NAME: 'monitoring' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - test monitoring pull +# - test monitoring run +# - test monitoring pytest +# script: +# - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/$IMAGE_NAME.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"' +# when: manual -# Deployment of the monitoring service in development Kubernetes Cluster -deploy monitoring development: - variables: - IMAGE_NAME: 'monitoring' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: deploy - needs: - - test monitoring pull - - test monitoring run - - test monitoring pytest - script: - - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' - - kubectl version - - kubectl get all - - kubectl apply -f "manifests/$IMAGE_NAME.yaml" - - kubectl get all - # environment: - # name: development - # url: https://example.com - # kubernetes: - # namespace: development - rules: - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - when: manual +# # Deployment of the monitoring service in development Kubernetes Cluster +# deploy monitoring development: +# variables: +# IMAGE_NAME: 'monitoring' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - test monitoring pull +# - test monitoring run +# - test monitoring pytest +# script: +# - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/$IMAGE_NAME.yaml" +# - kubectl get all +# # environment: +# # name: development +# # url: https://example.com +# # kubernetes: +# # namespace: development +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual -# Deployment of the monitoring service in production Kubernetes Cluster -deploy monitoring production: - variables: - IMAGE_NAME: 'monitoring' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: deploy - needs: - - test monitoring pull - - test monitoring run - - test monitoring pytest - script: - - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' - - kubectl version - - kubectl get all - - kubectl apply -f "manifests/$IMAGE_NAME.yaml" - - kubectl get all - # environment: - # name: production - # url: https://example.com - # kubernetes: - # namespace: production - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' - when: manual +# # Deployment of the monitoring service in production Kubernetes Cluster +# deploy monitoring production: +# variables: +# IMAGE_NAME: 'monitoring' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - test monitoring pull +# - test monitoring run +# - test monitoring pytest +# script: +# - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/$IMAGE_NAME.yaml" +# - kubectl get all +# # environment: +# # name: production +# # url: https://example.com +# # kubernetes: +# # namespace: production +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' +# when: manual diff --git a/src/monitoring/Dockerfile b/src/monitoring/Dockerfile index 1a7a289eb..e274b0a51 100644 --- a/src/monitoring/Dockerfile +++ b/src/monitoring/Dockerfile @@ -32,16 +32,4 @@ RUN python3 -m pip install -r requirements.txt COPY monitoring/. monitoring COPY common/logger.py common -# set listen port -ENV PORT=7070 -ARG user -ARG pass -ARG db -ARG hostname -ENV INFLUXDB_HOSTNAME=$hostname -ENV INFLUXDB_USER=$user -ENV INFLUXDB_PASSWORD=$pass -ENV INFLUXDB_DATABASE=$db -EXPOSE 7070 - ENTRYPOINT ["python", "-m", "monitoring.service"] diff --git a/src/monitoring/service/MonitoringServiceServicerImpl.py b/src/monitoring/service/MonitoringServiceServicerImpl.py index 1747e3a2b..daa9e4289 100644 --- a/src/monitoring/service/MonitoringServiceServicerImpl.py +++ b/src/monitoring/service/MonitoringServiceServicerImpl.py @@ -30,6 +30,11 @@ class MonitoringServiceServicerImpl(monitoring_pb2_grpc.MonitoringServiceService # Create influx_db client self.influx_db = influx_tools.Influx(INFLUXDB_HOSTNAME,"8086",INFLUXDB_USER,INFLUXDB_PASSWORD,INFLUXDB_DATABASE) + LOGGER.info(INFLUXDB_HOSTNAME) + LOGGER.info(INFLUXDB_USER) + LOGGER.info(INFLUXDB_PASSWORD) + LOGGER.info(INFLUXDB_DATABASE) + self.influx_db.read_KPI_points() # CreateKpi (CreateKpiRequest) returns (KpiId) {} def CreateKpi(self, request : monitoring_pb2.CreateKpiRequest, context) -> monitoring_pb2.KpiId : diff --git a/src/monitoring/tests/Dockerfile b/src/monitoring/tests/Dockerfile index 3c9b11e32..80c3c5e88 100644 --- a/src/monitoring/tests/Dockerfile +++ b/src/monitoring/tests/Dockerfile @@ -31,19 +31,5 @@ RUN python3 -m pip install -r requirements.txt COPY monitoring/. monitoring COPY common/logger.py common -# define variables for the influxdb -# ARG user -# ARG pass -# ARG db -# ARG hostname -# ENV INFLUXDB_HOSTNAME=$hostname -# ENV INFLUXDB_USER=$user -# ENV INFLUXDB_PASSWORD=$pass -# ENV INFLUXDB_DATABASE=$db - -# set listen port -ENV PORT=7070 -EXPOSE 7070 - #RUN pytest --junitxml=report.xml ENTRYPOINT ["pytest", "--junitxml=/opt/results/report.xml"] \ No newline at end of file -- GitLab