Commit 5db8c30d authored by Sergio Gonzalez Diaz's avatar Sergio Gonzalez Diaz
Browse files

Test monitoring component

parent df989308

manifests/monitoring.yaml

+9 −9
Original line number Diff line number Diff line
@@ -30,12 +30,12 @@ spec: 
        envFrom:

          - secretRef:

              name: influxdb-secrets

        # readinessProbe:

        #   exec:

        #     command: ["curl", "-XGET", "localhost:8086/health"]

        # livenessProbe:

        #   exec:

        #     command: ["curl", "-XGET", "localhost:8086/health"]

        readinessProbe:

          exec:

            command: ["curl", "-XGET", "localhost:8086/health"]

        livenessProbe:

          exec:

            command: ["curl", "-XGET", "localhost:8086/health"]

        resources:

          requests:

            cpu: 250m
@@ -48,9 +48,9 @@ spec: 
        imagePullPolicy: Always

        ports:

        - containerPort: 7070

        env:

        - name: PORT

          value: "7070"

        envFrom:

          - secretRef:

              name: monitoring-secrets

        readinessProbe:

          exec:

            command: ["/bin/grpc_health_probe", "-addr=:7070"]

src/monitoring/.gitlab-ci.yml

+133 −123
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ build monitoring: 
  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker build --build-arg hostname=localhost --build-arg user=$INFLUXDB_USER --build-arg pass=$INFLUXDB_PASSWORD --build-arg db=$INFLUXDB_DATABASE -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  after_script:
@@ -23,56 +23,67 @@ build monitoring: 
      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be pulled from the gitlab registry

test monitoring pull:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:

    - build monitoring

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

test kubernetes:

  stage: test

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

    - kubectl create secret generic monitoring-secrets --from-literal=INFLUXDB_DATABASE="monitoring" --from-literal=INFLUXDB_USER="teraflow" --from-literal=INFLUXDB_PASSWORD="teraflow" --from-literal=INFLUXDB_HOSTNAME="localhost"

    - kubectl describe secret monitoring-secrets

    - kubectl get all

    - kubectl apply -f "manifests/monitoring.yaml"

    - kubectl get all

    # - kubectl --namespace gitlab-ci logs monitoring-5f946cdc5d-99sjp influxdb

    # - kubectl --namespace gitlab-ci logs monitoring-5f946cdc5d-99sjp server



# test if the Docker image can be executed

test monitoring run:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:

    - build monitoring

  before_script:

    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

    - if docker ps | grep monitoring; then docker stop monitoring; else echo "monitoring is not running"; fi

  script:

    - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker ps

  after_script:

    - docker stop "$IMAGE_NAME"

    - docker network rm teraflowbridge

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

# # test if the Docker image can be pulled from the gitlab registry

# test monitoring pull:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: unit_test

#   needs:

#     - build monitoring

#   before_script:

#     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

#   script:

#     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml



# # test if the Docker image can be executed

# test monitoring run:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: unit_test

#   needs:

#     - build monitoring

#   before_script:

#     - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

#     - if docker ps | grep monitoring; then docker stop monitoring; else echo "monitoring is not running"; fi

#   script:

#     - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#     - docker ps

#   after_script:

#     - docker stop "$IMAGE_NAME"

#     - docker network rm teraflowbridge

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml



# apply unit test to the monitoring component

test monitoring pytest:
@@ -88,9 +99,8 @@ test monitoring pytest: 
    - if docker container ls | grep influxdb; then docker rm influxdb; fi

  script:

    - docker run --name influxdb -d -p 8086:8086 -e INFLUXDB_DB=$INFLUXDB_DATABASE -e INFLUXDB_ADMIN_USER=$INFLUXDB_USER -e INFLUXDB_ADMIN_PASSWORD=$INFLUXDB_PASSWORD -e INXLUXDB_HTTP_AUTH_ENABLED=True --network=teraflowbridge influxdb:1.8

    # - docker build --build-arg hostname=influxdb --build-arg user=$INFLUXDB_USER --build-arg pass=$INFLUXDB_PASSWORD --build-arg db=$INFLUXDB_DATABASE -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

    - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

    - docker run --name $IMAGE_NAME_TEST --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge  $IMAGE_NAME_TEST:$IMAGE_TAG

    - docker run --name $IMAGE_NAME_TEST -p 7070:7070 --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge  $IMAGE_NAME_TEST:$IMAGE_TAG

  after_script:

    - docker stop influxdb

    - docker rm influxdb
@@ -111,77 +121,77 @@ test monitoring pytest: 
      reports:

        junit: src/$IMAGE_NAME/tests/report.xml



# Deployment of the monitoring service in testing Kubernetes Cluster

deploy monitoring test:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: test

  #   url: https://example.com

  #   kubernetes:

  #     namespace: test

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

      when: manual

# # Deployment of the monitoring service in testing Kubernetes Cluster

# deploy monitoring test:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: test

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: test

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

#       when: manual



# Deployment of the monitoring service in development Kubernetes Cluster

deploy monitoring development:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: development

  #   url: https://example.com

  #   kubernetes:

  #     namespace: development

  rules:

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

      when: manual

# # Deployment of the monitoring service in development Kubernetes Cluster

# deploy monitoring development:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: development

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: development

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

#       when: manual



# Deployment of the monitoring service in production Kubernetes Cluster

deploy monitoring production:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: production

  #   url: https://example.com

  #   kubernetes:

  #     namespace: production

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

      when: manual
 
# # Deployment of the monitoring service in production Kubernetes Cluster

# deploy monitoring production:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: production

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: production

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

#       when: manual

src/monitoring/Dockerfile

+0 −12
Original line number Diff line number Diff line
@@ -32,16 +32,4 @@ RUN python3 -m pip install -r requirements.txt 
COPY monitoring/. monitoring

COPY common/logger.py common



# set listen port

ENV PORT=7070

ARG user

ARG pass

ARG db

ARG hostname

ENV INFLUXDB_HOSTNAME=$hostname

ENV INFLUXDB_USER=$user

ENV INFLUXDB_PASSWORD=$pass

ENV INFLUXDB_DATABASE=$db

EXPOSE 7070



ENTRYPOINT ["python", "-m", "monitoring.service"]

src/monitoring/service/MonitoringServiceServicerImpl.py

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,11 @@ class MonitoringServiceServicerImpl(monitoring_pb2_grpc.MonitoringServiceService 


        # Create influx_db client

        self.influx_db = influx_tools.Influx(INFLUXDB_HOSTNAME,"8086",INFLUXDB_USER,INFLUXDB_PASSWORD,INFLUXDB_DATABASE)

        LOGGER.info(INFLUXDB_HOSTNAME)

        LOGGER.info(INFLUXDB_USER)

        LOGGER.info(INFLUXDB_PASSWORD)

        LOGGER.info(INFLUXDB_DATABASE)

        self.influx_db.read_KPI_points()



    # CreateKpi (CreateKpiRequest) returns (KpiId) {}

    def CreateKpi(self, request : monitoring_pb2.CreateKpiRequest, context) -> monitoring_pb2.KpiId :

src/monitoring/tests/Dockerfile

+0 −14
Original line number Diff line number Diff line
@@ -31,19 +31,5 @@ RUN python3 -m pip install -r requirements.txt 
COPY monitoring/. monitoring

COPY common/logger.py common



# define variables for the influxdb

# ARG user

# ARG pass

# ARG db

# ARG hostname

# ENV INFLUXDB_HOSTNAME=$hostname

# ENV INFLUXDB_USER=$user

# ENV INFLUXDB_PASSWORD=$pass

# ENV INFLUXDB_DATABASE=$db



# set listen port

ENV PORT=7070

EXPOSE 7070



#RUN pytest --junitxml=report.xml

ENTRYPOINT ["pytest", "--junitxml=/opt/results/report.xml"]
 
 No newline at end of file