Commit ab2131ca authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/fix-cicd-context-device-service-compute' into 'develop' 

Multiple adjustments in CI/CD pipeline and related issues.

See merge request teraflow-h2020/controller!45
parents c2e535a0 7f5119da

.gitlab-ci.yml

+7 −7
Original line number Diff line number Diff line
@@ -12,13 +12,13 @@ stages: 
include:

  - local: '/manifests/.gitlab-ci.yml'

  - local: '/src/monitoring/.gitlab-ci.yml'

  - local: '/src/centralizedattackdetector/.gitlab-ci.yml'

  #- local: '/src/centralizedattackdetector/.gitlab-ci.yml'

  - local: '/src/context/.gitlab-ci.yml'

  - local: '/src/device/.gitlab-ci.yml'

  - local: '/src/service/.gitlab-ci.yml'

  - local: '/src/tester_integration/.gitlab-ci.yml'

  - local: '/src/tester_functional/.gitlab-ci.yml'

  - local: '/src/automation/.gitlab-ci.yml'

  - local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'

  - local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml'

  - local: '/src/l3_attackmitigator/.gitlab-ci.yml'
 
  #- local: '/src/tester_integration/.gitlab-ci.yml'

  #- local: '/src/tester_functional/.gitlab-ci.yml'

  #- local: '/src/automation/.gitlab-ci.yml'

  #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'

  #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml'

  #- local: '/src/l3_attackmitigator/.gitlab-ci.yml'

proto/context.proto

+1 −1
Original line number Diff line number Diff line
@@ -243,6 +243,7 @@ message EndPointId { 
message EndPoint {

  EndPointId endpoint_id = 1;

  string endpoint_type = 2;

  repeated kpi_sample_types.KpiSampleType kpi_sample_types = 3;

}
@@ -257,7 +258,6 @@ message ConfigRule { 
  ConfigActionEnum action = 1;

  string resource_key = 2;

  string resource_value = 3;

  kpi_sample_types.KpiSampleType kpi_sample_type = 4;

}

proto/kpi_sample_types.proto

+5 −5
Original line number Diff line number Diff line
@@ -2,9 +2,9 @@ syntax = "proto3"; 
package kpi_sample_types;



enum KpiSampleType {

    UNKNOWN = 0;

    PACKETS_TRANSMITTED = 101;

    PACKETS_RECEIVED    = 102;

    BYTES_TRANSMITTED   = 201;

    BYTES_RECEIVED      = 202;

    KPISAMPLETYPE_UNKNOWN = 0;

    KPISAMPLETYPE_PACKETS_TRANSMITTED = 101;

    KPISAMPLETYPE_PACKETS_RECEIVED    = 102;

    KPISAMPLETYPE_BYTES_TRANSMITTED   = 201;

    KPISAMPLETYPE_BYTES_RECEIVED      = 202;

}

src/common/type_checkers/Assertions.py

+14 −1
Original line number Diff line number Diff line
@@ -27,6 +27,16 @@ def validate_device_operational_status_enum(message): 
        'DEVICEOPERATIONALSTATUS_ENABLED'

    ]



def validate_kpi_sample_types_enum(message):

    assert isinstance(message, str)

    assert message in [

        'KPISAMPLETYPE_UNKNOWN',

        'KPISAMPLETYPE_PACKETS_TRANSMITTED',

        'KPISAMPLETYPE_PACKETS_RECEIVED',

        'KPISAMPLETYPE_BYTES_TRANSMITTED',

        'KPISAMPLETYPE_BYTES_RECEIVED',

    ]



def validate_service_type_enum(message):

    assert isinstance(message, str)

    assert message in [
@@ -222,11 +232,14 @@ def validate_topology(message, num_devices=None, num_links=None): 


def validate_endpoint(message):

    assert isinstance(message, dict)

    assert len(message.keys()) == 2

    assert len(message.keys()) == 3

    assert 'endpoint_id' in message

    validate_endpoint_id(message['endpoint_id'])

    assert 'endpoint_type' in message

    assert isinstance(message['endpoint_type'], str)

    assert 'kpi_sample_types' in message

    assert isinstance(message['kpi_sample_types'], list)

    for kpi_sample_type in message['kpi_sample_types']: validate_kpi_sample_types_enum(kpi_sample_type)



def validate_device(message):

    assert isinstance(message, dict)

src/compute/.gitlab-ci.yml

+40 −19
Original line number Diff line number Diff line 
# Build, tag, and push the Docker images to the GitLab Docker registry

# Build, tag and push the Docker image to the GitLab registry

build compute:

  variables:

    IMAGE_NAME: 'compute' # name of the microservice

    IMAGE_NAME_TEST: 'compute-test' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: build

  before_script:
@@ -11,16 +10,22 @@ build compute: 
    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

#  after_script:

#    - docker rmi $(docker images --quiet --filter=dangling=true)

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - manifests/$IMAGE_NAMEservice.yaml

      - .gitlab-ci.yml



# Pull, execute, and run unitary tests for the Docker image from the GitLab registry

unit_test compute:

# Apply unit test to the component

unit test compute:

  variables:

    IMAGE_NAME: 'compute' # name of the microservice

    IMAGE_NAME_TEST: 'compute-test' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: unit_test

  needs:
@@ -28,33 +33,49 @@ unit_test compute: 
  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi

    - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker run -d -p 9090:9090 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG"

    - docker ps -a

    - docker run --name $IMAGE_NAME -d -p 9090:9090 --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG

    - sleep 5

    - docker ps -a

    - docker logs $IMAGE_NAME

    - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py"

  after_script:

    - docker stop $IMAGE_NAME

    - docker rm $IMAGE_NAME

    - docker rm -f $IMAGE_NAME

    - docker network rm teraflowbridge

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 

    - changes:

      - src/$IMAGE_NAME/**

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

      - src/$IMAGE_NAME/tests/Dockerfile

      - manifests/$IMAGE_NAMEservice.yaml

      - .gitlab-ci.yml



# Deployment of the service in Kubernetes Cluster

deploy compute:

  variables:

    IMAGE_NAME: 'compute' # name of the microservice

    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)

  stage: deploy

  needs:

    - build compute

    - unit_test compute

    - dependencies all

    - integ_test execute

    - unit test compute

    # - integ_test execute

  script:

    - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/$IMAGE_NAMEservice.yaml'

    - kubectl version

    - kubectl get all

    - kubectl apply -f "manifests/computeservice.yaml"

    - kubectl delete pods --selector app=computeservice

    - kubectl apply -f "manifests/$IMAGE_NAMEservice.yaml"

    - kubectl get all

  # environment:

  #   name: test

  #   url: https://example.com

  #   kubernetes:

  #     namespace: test

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

      when: manual    

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

      when: manual