Commit 55e4b397 authored by Sergio Gonzalez Diaz's avatar Sergio Gonzalez Diaz
Browse files

Update monitoring service with influxdb

parent 4aa413f2

manifests/influx.yaml

deleted100644 → 0
+0 −63
Original line number Diff line number Diff line 
---

apiVersion: apps/v1

kind: Deployment

metadata:

  name: influxdb

spec:

  selector:

    matchLabels:

      app: influxdb

  replicas: 1

  template:

    metadata:

      labels:

        app: influxdb

        version: v1

    spec:

      containers:

      - name: influxdb

        image: influxdb:1.8

        ports:

        - containerPort: 8086

        # env:

        # - name: INFLUXDB_DB

        #   value: "monitoring"

        # - name: INFLUXDB_ADMIN_USER

        #   value: "teraflow"

        # - name: INFLUXDB_ADMIN_PASSWORD

        #   value: "teraflow"

        # - name: INFLUXDB_HTTP_AUTH_ENABLED

        #   value: "True"

---

apiVersion: v1

kind: Service

metadata:

  name: influxdb

  labels:

    app: influxdb

spec:

  type: ClusterIP

  selector:

    app: influxdb

  ports:

  - name: influxdb

    protocol: TCP

    port: 8086

    targetPort: 8086

---

apiVersion: v1

kind: Service

metadata:

  name: influxdb-public

  labels:

    app: influxdb

spec:

  type: NodePort

  selector:

    app: influxdb

  ports:

  - name: influxdb

    protocol: TCP

    port: 8086

    targetPort: 8086

---
 
 No newline at end of file

manifests/monitoring.yaml

+1 −29
Original line number Diff line number Diff line
@@ -21,15 +21,6 @@ spec: 
        envFrom:

          - secretRef:

              name: influxdb-secrets

        # env:

        # - name: INFLUXDB_DB

        #   value: "monitoring"

        # - name: INFLUXDB_ADMIN_USER

        #   value: "teraflow"

        # - name: INFLUXDB_ADMIN_PASSWORD

        #   value: "teraflow"

        # - name: INFLUXDB_HTTP_AUTH_ENABLED

        #   value: "True"

        readinessProbe:

          exec:

            command: ["curl", "-XGET", "localhost:8086/health"]
@@ -64,8 +55,6 @@ spec: 
          limits:

            cpu: 700m

            memory: 1024Mi

      # imagePullSecrets:

      # - name: regcred

---

apiVersion: v1

kind: Service
@@ -85,20 +74,3 @@ spec: 
    port: 8086

    targetPort: 8086

---
 
 No newline at end of file
 
apiVersion: v1

kind: Service

metadata:

  name: monitoring-public

spec:

  type: NodePort

  selector:

    app: monitoring

  ports:

  - name: grpc

    protocol: TCP

    port: 7070

    targetPort: 7070

  - name: influxdb

    protocol: TCP

    port: 8086

    targetPort: 8086
 
 No newline at end of file

src/monitoring/.gitlab-ci.yml

+160 −193
Original line number Diff line number Diff line 
# build, tag and push the Docker image to the gitlab registry

# build monitoring:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: build

#   before_script:

#     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

#   script:

#     - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

#     - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#     - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#   after_script:

#     - docker rmi $(docker images --quiet --filter=dangling=true)

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml



build influx:

build monitoring:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: build

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    # - kubectl version

    # - kubectl get all

    # # - kubectl delete -f "manifests/monitoring.yaml"

    # # - sleep 15

    # # - kubectl get all

    # - kubectl apply -f "manifests/monitoring.yaml"

    # - sleep 60

    # # - kubectl get all

    # # - kubectl apply -f "manifests/influx.yaml"

    # # - sleep 15

    # # - kubectl get all

    # - curl -XGET "10.97.74.29:8086/health" 

    # # - kubectl delete -f "manifests/influx.yaml"

    # - kubectl delete secret influxdb-secrets

    # - kubectl create secret generic influxdb-secrets --from-literal=INFLUXDB_DB="monitoring" --from-literal=INFLUXDB_ADMIN_USER="teraflow" --from-literal=INFLUXDB_ADMIN_PASSWORD="teraflow" --from-literal=INFLUXDB_HTTP_AUTH_ENABLED="True"

    # - kubectl describe secret influxdb-secrets

    # - kubectl delete -f "manifests/monitoring.yaml"

    # - sleep 15

    # - kubectl get all

    # - kubectl apply -f "manifests/monitoring.yaml"

    # - sleep 60

    # - kubectl get all

    # - cd ./src/monitoring/service

    # - docker build -t "test-influx" ./

    # - docker run --name test-influx --rm test-influx

    # - docker rmi test-influx

    - kubectl --namespace gitlab-ci describe pod monitoring-9df446bb5-4wr56

    - kubectl --namespace gitlab-ci logs monitoring-9df446bb5-4wr56 server

    - kubectl --namespace gitlab-ci logs monitoring-9df446bb5-4wr56 influxdb

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  after_script:

    - docker rmi $(docker images --quiet --filter=dangling=true)

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be pulled from the gitlab registry

# test monitoring pull:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: unit_test

#   needs:

#     - build monitoring

#   before_script:

#     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

#   script:

#     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml

test monitoring pull:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:

    - build monitoring

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be executed

# test monitoring run:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: unit_test

#   needs:

#     - build monitoring

#   before_script:

#     - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

#   script:

#     - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#     - docker ps

#   after_script:

#     - docker stop "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml

test monitoring run:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:

    - build monitoring

  before_script:

    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

  script:

    - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker ps

  after_script:

    - docker stop "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# apply unit test to the monitoring component

# test monitoring pytest:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_NAME_TEST: 'monitoring-test' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: unit_test

#   needs:

#     - build monitoring

#   script:

#     - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

#     - docker run -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" $IMAGE_NAME_TEST:$IMAGE_TAG

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

#     - changes:

#       - src/$IMAGE_NAME/**/*.{py,in,yml}

#       - src/$IMAGE_NAME/Dockerfile

#       - src/$IMAGE_NAME/tests/*.py

#       - src/$IMAGE_NAME/tests/Dockerfile

#       - manifests/$IMAGE_NAME.yaml

#       - .gitlab-ci.yml

#   artifacts:

#       when: always

#       reports:

#         junit: src/$IMAGE_NAME/tests/report.xml

test monitoring pytest:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_NAME_TEST: 'monitoring-test' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:

    - build monitoring

  script:

    - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

    - docker run -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" $IMAGE_NAME_TEST:$IMAGE_TAG

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

  artifacts:

      when: always

      reports:

        junit: src/$IMAGE_NAME/tests/report.xml



# Deployment of the monitoring service in testing Kubernetes Cluster

# deploy monitoring test:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: test

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: test

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

#       when: manual

deploy monitoring test:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: test

  #   url: https://example.com

  #   kubernetes:

  #     namespace: test

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

      when: manual



# Deployment of the monitoring service in development Kubernetes Cluster

# deploy monitoring development:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: development

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: development

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

#       when: manual

deploy monitoring development:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: development

  #   url: https://example.com

  #   kubernetes:

  #     namespace: development

  rules:

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

      when: manual



# Deployment of the monitoring service in production Kubernetes Cluster

# deploy monitoring production:

#   variables:

#     IMAGE_NAME: 'monitoring' # name of the microservice

#     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

#   stage: deploy

#   needs:

#     - test monitoring pull

#     - test monitoring run

#     - test monitoring pytest

#   script:

#     - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

#     - kubectl version

#     - kubectl get all

#     - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

#     - kubectl get all

#   # environment:

#   #   name: production

#   #   url: https://example.com

#   #   kubernetes:

#   #     namespace: production

#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

#       when: manual
 
deploy monitoring production:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - test monitoring pull

    - test monitoring run

    - test monitoring pytest

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: production

  #   url: https://example.com

  #   kubernetes:

  #     namespace: production

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

      when: manual

src/monitoring/service/Dockerfile

deleted100644 → 0
+0 −8
Original line number Diff line number Diff line 
FROM python:3-slim



RUN pip install --upgrade pip 

RUN pip install influxdb

RUN pip install datetime

COPY ./test_influx.py .

COPY ./influx_tools.py .

ENTRYPOINT ["python3", "test_influx.py"]
 
 No newline at end of file

src/monitoring/service/test_influx.py

deleted100644 → 0
+0 −8
Original line number Diff line number Diff line 
from influx_tools import *

from datetime import datetime



print("Starting")

influx = Influx('10.108.110.24', 8086, 'teraflow', 'teraflow', 'monitoring')

influx.write_KPI(datetime.now(),1,1,101,1000)

results = influx.client.query('select * from samples;')

print(results.raw)
 
 No newline at end of file