Commit e87d5218 authored by Sergio Gonzalez Diaz's avatar Sergio Gonzalez Diaz
Browse files

Merge branch 'feat/kubernetes-integration' into 'develop' 

Merge feat/kubernetes-integration into develop

See merge request teraflow-h2020/controller!15
parents 0ecb9820 6f05db20

manifests/monitoringservice.yaml→manifests/monitoring.yaml

+6 −6
Original line number Diff line number Diff line 
apiVersion: apps/v1

kind: Deployment

metadata:

  name: monitoringservice

  name: monitoring

spec:

  selector:

    matchLabels:

      app: monitoringservice

      app: monitoring

  template:

    metadata:

      labels:

        app: monitoringservice

        app: monitoring

    spec:

      terminationGracePeriodSeconds: 5

      containers:

      - name: server

        image: myregistrydomain.com/monitoringservice

        image: registry.gitlab.com/teraflow-h2020/controller/monitoring:latest

        ports:

        - containerPort: 8080

        env:
@@ -39,11 +39,11 @@ spec: 
apiVersion: v1

kind: Service

metadata:

  name: monitoringservice

  name: monitoring

spec:

  type: ClusterIP

  selector:

    app: monitoringservice

    app: monitoring

  ports:

  - name: grpc

    protocol: TCP

src/gitlab-ci.yml_generator.py

0 → 100755
+180 −0
Original line number Diff line number Diff line 
#!/usr/bin/python

import argparse

import os.path



# Parse the arguments

parser = argparse.ArgumentParser(description='Generate .gitlab-cy.yml template for a TeraFlow microservice.')

parser.add_argument("microservice", help="name of your microservice", type=str)

parser.add_argument("-t", "--tag", help="tag of the microservice Docker container", type=str, default='latest', required=False)

args = parser.parse_args()



# Check if the file and the path already exists

path="./{microservice}".format(microservice = args.microservice)

file="{path}/.gitlab-ci.yml".format(path = path)

if(os.path.isfile(file)):

    if input("File already exists, do you want to overwrite? (y/n) ") != "y":

        exit()

if(os.path.lexists(path)!= True):

    try:

        os.mkdir(path)

    except OSError:

        print ("Creation of the directory %s failed" % path)

    else:

        print ("Successfully created the directory %s " % path)



# Create the gitlab-ci.yml template file

f=open(file,"w+")

yml_template = """

# build, tag and push the Docker image to the gitlab registry

build {microservice}:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: build

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - changes:

      - src/$IMAGE_NAME/*.{{py,in,yml}}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be pulled from the gitlab registry

test {microservice} pull:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - build {microservice}

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - changes:

      - src/$IMAGE_NAME/*.{{py,in,yml}}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be executed

test {microservice} run:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - build {microservice}

  before_script:

    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

  script:

    - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$IMAGE_NAME:$IMAGE_TAG"

    - docker ps

  after_script:

    - docker stop {microservice}

  rules:

    - changes:

      - src/$IMAGE_NAME/*.{{py,in,yml}}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# apply unit test to the {microservice} component

test {microservice} pytest:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_NAME_TEST: '{microservice}-test' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - build {microservice}

  script:

    - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

    - docker run -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" $IMAGE_NAME_TEST:$IMAGE_TAG

  rules:

    - changes:

      - src/$IMAGE_NAME/*.{{py,in,yml}}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

  artifacts:

      when: always

      reports:

        junit: src/$IMAGE_NAME/tests/report.xml



# Deployment of the {microservice} service in development Kubernetes Cluster

deploy {microservice} development:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - build {microservice}

    - test {microservice} run

  script:

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: development

  #   url: https://example.com

  #   kubernetes:

  #     namespace: development

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

      when: manual

      changes:

        - src/$IMAGE_NAME/*.{{py,in,yml}}

        - src/$IMAGE_NAME/Dockerfile

        - src/$IMAGE_NAME/tests/*.py

        - src/$IMAGE_NAME/tests/Dockerfile

        - manifests/$IMAGE_NAME.yaml

        - .gitlab-ci.yml



# Deployment of the {microservice} service in production Kubernetes Cluster

deploy {microservice} development:

  variables:

    IMAGE_NAME: '{microservice}' # name of the microservice

    IMAGE_TAG: '{tag}' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - build {microservice}

    - test {microservice} run

  script:

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: production

  #   url: https://example.com

  #   kubernetes:

  #     namespace: production

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

      when: manual

      changes:

        - src/$IMAGE_NAME/*.{{py,in,yml}}

        - src/$IMAGE_NAME/Dockerfile

        - src/$IMAGE_NAME/tests/*.py

        - src/$IMAGE_NAME/tests/Dockerfile

        - manifests/$IMAGE_NAME.yaml

        - .gitlab-ci.yml

"""

f.write(yml_template.format(microservice = args.microservice, tag=args.tag))

print("File created in the following path: {file}".format(file=file))

src/monitoring/.gitlab-ci.yml

+95 −48
Original line number Diff line number Diff line 
# build, tag and push the Docker image to the gitlab registry

build monitoring:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

  IMAGE_NAME_TEST: 'monitoring-test' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)



# build the Docker image

build monitoring:

  stage: build

  script:

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - .gitlab-ci.yml



# tags the Docker image

tag monitoring:

  stage: build

  script:

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - .gitlab-ci.yml



# push the Docker image to the gitlab Docker registry

push monitoring:

  stage: build

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml





# test if the Docker image can be pulled from the gitlab registry

test monitoring pull:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - push monitoring

    - build monitoring

  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml



# test if the Docker image can be executed

test monitoring run:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - build monitoring

  before_script:

    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  

  script:

    - docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge --rm "$IMAGE_NAME:$IMAGE_TAG"

    - docker ps > deploy_test_report.txt

    - docker run -d -p 7070:7070 --name $IMAGE_NAME --network=teraflowbridge --rm "$IMAGE_NAME:$IMAGE_TAG"

    - docker ps

  after_script:

    - docker stop monitoring

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

  artifacts:

    when: always

    paths:

      - deploy_test_report.txt

    expire_in: 1 day



# apply unit test to the monitoring component

test monitoring pytest:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_NAME_TEST: 'monitoring-test' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: test

  needs:

    - build monitoring

  script:

    - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/ > pytest_report.txt

    - docker build -t "$IMAGE_NAME_TEST:$IMAGE_TAG" -f ./src/$IMAGE_NAME/tests/Dockerfile ./src/

    - docker run -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" $IMAGE_NAME_TEST:$IMAGE_TAG

  rules:

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAME.yaml

      - .gitlab-ci.yml

  artifacts:

      when: always

      paths:

        - pytest_report.txt

      expire_in: 1 day

      reports:

        junit: src/$IMAGE_NAME/tests/report.xml



# Deployment of the monitoring service in Kubernetes Cluster

deploy monitoring:

# Deployment of the monitoring service in development Kubernetes Cluster

deploy monitoring development:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - build monitoring

    - test monitoring run

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/monitoringservice.yaml"

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: development

  #   url: https://example.com

  #   kubernetes:

  #     namespace: development

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'

      when: manual

      changes:

        - src/$IMAGE_NAME/*.{py,in,yml}

        - src/$IMAGE_NAME/Dockerfile

        - src/$IMAGE_NAME/tests/*.py

        - src/$IMAGE_NAME/tests/Dockerfile

        - manifests/$IMAGE_NAME.yaml

        - .gitlab-ci.yml



# Deployment of the monitoring service in production Kubernetes Cluster

deploy monitoring production:

  variables:

    IMAGE_NAME: 'monitoring' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - build monitoring

    - test monitoring run

  script:

    - 'sed -i "s/image: .*/image: $CI_REGISTRY\/$CI_PROJECT_NAMESPACE\/$CI_PROJECT_NAME\/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAME.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"

    - kubectl get all

  # environment:

  #   name: production

  #   url: https://example.com

  #   kubernetes:

  #     namespace: production

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'

      when: manual

      changes:

        - src/$IMAGE_NAME/*.{py,in,yml}

        - src/$IMAGE_NAME/Dockerfile

        - src/$IMAGE_NAME/tests/*.py

        - src/$IMAGE_NAME/tests/Dockerfile

        - manifests/$IMAGE_NAME.yaml

        - .gitlab-ci.yml

src/monitoring/tests/Dockerfile

+2 −1
Original line number Diff line number Diff line
@@ -27,4 +27,5 @@ COPY common/logger.py . 
ENV PORT=7070

EXPOSE 7070



RUN pytest --junitxml=report.xml
 
#RUN pytest --junitxml=report.xml

ENTRYPOINT ["pytest", "--junitxml=/opt/results/report.xml"]