From 153c38448df017e2dcc92291178ab4a42b0c7103 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Mon, 10 Jan 2022 16:21:19 +0000 Subject: [PATCH] Fix integration of UPM services into GitLab CI pipeline --- .gitlab-ci.yml | 6 +- manifests/l3_attackmitigatorservice.yaml | 12 ++-- .../l3_centralizedattackdetectorservice.yaml | 12 ++-- .../l3_distributedattackdetectorservice.yaml | 12 ++-- src/l3_attackmitigator/.gitlab-ci.yml | 55 +++++++++++++------ .../.gitlab-ci.yml | 55 +++++++++++++------ .../.gitlab-ci.yml | 54 ++++++++++++------ 7 files changed, 135 insertions(+), 71 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8f78f4df..0489c5971 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,6 @@ include: #- local: '/src/tester_integration/.gitlab-ci.yml' #- local: '/src/tester_functional/.gitlab-ci.yml' #- local: '/src/automation/.gitlab-ci.yml' - #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' - #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' - #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' + - local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' + - local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' + - local: '/src/l3_attackmitigator/.gitlab-ci.yml' diff --git a/manifests/l3_attackmitigatorservice.yaml b/manifests/l3_attackmitigatorservice.yaml index 7ac91b02f..34f660324 100644 --- a/manifests/l3_attackmitigatorservice.yaml +++ b/manifests/l3_attackmitigatorservice.yaml @@ -1,20 +1,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: attackmitigatorservice + name: l3_attackmitigatorservice spec: selector: matchLabels: - app: attackmitigatorservice + app: l3_attackmitigatorservice template: metadata: labels: - app: attackmitigatorservice + app: l3_attackmitigatorservice spec: terminationGracePeriodSeconds: 5 containers: - name: server - image: registry.gitlab.com/teraflow-h2020/controller/attackmitigator:latest + image: registry.gitlab.com/teraflow-h2020/controller/l3_attackmitigator:latest imagePullPolicy: Always ports: - containerPort: 10002 @@ -38,11 +38,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: attackmitigatorservice + name: l3_attackmitigatorservice spec: type: ClusterIP selector: - app: attackmitigatorservice + app: l3_attackmitigatorservice ports: - name: grpc port: 10002 diff --git a/manifests/l3_centralizedattackdetectorservice.yaml b/manifests/l3_centralizedattackdetectorservice.yaml index 4b4b62929..0393d83c2 100644 --- a/manifests/l3_centralizedattackdetectorservice.yaml +++ b/manifests/l3_centralizedattackdetectorservice.yaml @@ -1,20 +1,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: centralizedattackdetectorservice + name: l3_centralizedattackdetectorservice spec: selector: matchLabels: - app: centralizedattackdetectorservice + app: l3_centralizedattackdetectorservice template: metadata: labels: - app: centralizedattackdetectorservice + app: l3_centralizedattackdetectorservice spec: terminationGracePeriodSeconds: 5 containers: - name: server - image: registry.gitlab.com/teraflow-h2020/controller/centralizedattackdetector:latest + image: registry.gitlab.com/teraflow-h2020/controller/l3_centralizedattackdetector:latest imagePullPolicy: Always ports: - containerPort: 10001 @@ -38,11 +38,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: centralizedattackdetectorservice + name: l3_centralizedattackdetectorservice spec: type: ClusterIP selector: - app: centralizedattackdetectorservice + app: l3_centralizedattackdetectorservice ports: - name: grpc port: 10001 diff --git a/manifests/l3_distributedattackdetectorservice.yaml b/manifests/l3_distributedattackdetectorservice.yaml index 22157d9a4..eff047b1b 100644 --- a/manifests/l3_distributedattackdetectorservice.yaml +++ b/manifests/l3_distributedattackdetectorservice.yaml @@ -1,20 +1,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: distributedattackdetectorservice + name: l3_distributedattackdetectorservice spec: selector: matchLabels: - app: distributedattackdetectorservice + app: l3_distributedattackdetectorservice template: metadata: labels: - app: distributedattackdetectorservice + app: l3_distributedattackdetectorservice spec: terminationGracePeriodSeconds: 5 containers: - name: server - image: registry.gitlab.com/teraflow-h2020/controller/distributedattackdetector:latest + image: registry.gitlab.com/teraflow-h2020/controller/l3_distributedattackdetector:latest imagePullPolicy: Always ports: - containerPort: 10000 @@ -38,11 +38,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: distributedattackdetectorservice + name: l3_distributedattackdetectorservice spec: type: ClusterIP selector: - app: distributedattackdetectorservice + app: l3_distributedattackdetectorservice ports: - name: grpc port: 10000 diff --git a/src/l3_attackmitigator/.gitlab-ci.yml b/src/l3_attackmitigator/.gitlab-ci.yml index e20771744..1fb3d88ee 100644 --- a/src/l3_attackmitigator/.gitlab-ci.yml +++ b/src/l3_attackmitigator/.gitlab-ci.yml @@ -2,7 +2,6 @@ build l3_attackmitigator: variables: IMAGE_NAME: 'l3_attackmitigator' # name of the microservice - IMAGE_NAME_TEST: 'l3_attackmitigator-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: @@ -11,16 +10,22 @@ build l3_attackmitigator: - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Pull, execute, and run unitary tests for the Docker image from the GitLab registry -unit_test l3_attackmitigator: +unit test l3_attackmitigator: variables: IMAGE_NAME: 'l3_attackmitigator' # name of the microservice - IMAGE_NAME_TEST: 'l3_attackmitigator-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: @@ -28,34 +33,50 @@ unit_test l3_attackmitigator: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run -d -p 10002:10002 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG" - - docker ps -a + - docker run --name $IMAGE_NAME -d -p 10002:10002 --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - - docker port $IMAGE_NAME - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose -o log_cli=true $IMAGE_NAME/tests/test_unitary.py" + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py" after_script: - - docker stop $IMAGE_NAME - - docker rm $IMAGE_NAME + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Deployment of the service in Kubernetes Cluster deploy l3_attackmitigator: + variables: + IMAGE_NAME: 'l3_attackmitigator' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: deploy needs: - - build l3_attackmitigator - - unit_test l3_attackmitigator - - dependencies all - - integ_test execute + - unit test l3_attackmitigator + # - integ_test execute script: + - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' - kubectl version - kubectl get all - - kubectl apply -f "manifests/l3_attackmitigatorservice.yaml" - - kubectl delete pods --selector app=l3_attackmitigatorservice + - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" - kubectl get all + # environment: + # name: test + # url: https://example.com + # kubernetes: + # namespace: test + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + when: manual + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + when: manual diff --git a/src/l3_centralizedattackdetector/.gitlab-ci.yml b/src/l3_centralizedattackdetector/.gitlab-ci.yml index aaebe01e4..cd410cb9c 100644 --- a/src/l3_centralizedattackdetector/.gitlab-ci.yml +++ b/src/l3_centralizedattackdetector/.gitlab-ci.yml @@ -2,7 +2,6 @@ build l3_centralizedattackdetector: variables: IMAGE_NAME: 'l3_centralizedattackdetector' # name of the microservice - IMAGE_NAME_TEST: 'l3_centralizedattackdetector-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: @@ -11,16 +10,22 @@ build l3_centralizedattackdetector: - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Pull, execute, and run unitary tests for the Docker image from the GitLab registry -unit_test l3_centralizedattackdetector: +unit test l3_centralizedattackdetector: variables: IMAGE_NAME: 'l3_centralizedattackdetector' # name of the microservice - IMAGE_NAME_TEST: 'l3_centralizedattackdetector-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: @@ -28,34 +33,50 @@ unit_test l3_centralizedattackdetector: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run -d -p 10001:10001 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG" - - docker ps -a + - docker run --name $IMAGE_NAME -d -p 10001:10001 --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - - docker port $IMAGE_NAME - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose -o log_cli=true $IMAGE_NAME/tests/test_unitary.py" + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py" after_script: - - docker stop $IMAGE_NAME - - docker rm $IMAGE_NAME + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Deployment of the service in Kubernetes Cluster deploy l3_centralizedattackdetector: + variables: + IMAGE_NAME: 'l3_centralizedattackdetector' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: deploy needs: - - build l3_centralizedattackdetector - - unit_test l3_centralizedattackdetector - - dependencies all - - integ_test execute + - unit test l3_centralizedattackdetector + # - integ_test execute script: + - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' - kubectl version - kubectl get all - - kubectl apply -f "manifests/l3_centralizedattackdetectorservice.yaml" - - kubectl delete pods --selector app=l3_centralizedattackdetectorservice + - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" - kubectl get all + # environment: + # name: test + # url: https://example.com + # kubernetes: + # namespace: test + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + when: manual + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + when: manual diff --git a/src/l3_distributedattackdetector/.gitlab-ci.yml b/src/l3_distributedattackdetector/.gitlab-ci.yml index c2579c70a..435e6ee83 100644 --- a/src/l3_distributedattackdetector/.gitlab-ci.yml +++ b/src/l3_distributedattackdetector/.gitlab-ci.yml @@ -2,7 +2,6 @@ build l3_distributedattackdetector: variables: IMAGE_NAME: 'l3_distributedattackdetector' # name of the microservice - IMAGE_NAME_TEST: 'l3_distributedattackdetector-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: @@ -11,16 +10,22 @@ build l3_distributedattackdetector: - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Pull, execute, and run unitary tests for the Docker image from the GitLab registry -unit_test l3_distributedattackdetector: +unit test l3_distributedattackdetector: variables: IMAGE_NAME: 'l3_distributedattackdetector' # name of the microservice - IMAGE_NAME_TEST: 'l3_distributedattackdetector-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: @@ -28,33 +33,50 @@ unit_test l3_distributedattackdetector: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run -d -p 10000:10000 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG" - - docker ps -a + - docker run --name $IMAGE_NAME -d -p 10000:10000 --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose -o log_cli=true $IMAGE_NAME/tests/test_unitary.py" + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py" after_script: - - docker stop $IMAGE_NAME - - docker rm $IMAGE_NAME + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/$IMAGE_NAME/** + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml # Deployment of the service in Kubernetes Cluster deploy l3_distributedattackdetector: + variables: + IMAGE_NAME: 'l3_distributedattackdetector' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: deploy needs: - - build l3_distributedattackdetector - - unit_test l3_distributedattackdetector - - dependencies all - - integ_test execute + - unit test l3_distributedattackdetector + # - integ_test execute script: + - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' - kubectl version - kubectl get all - - kubectl apply -f "manifests/l3_distributedattackdetectorservice.yaml" - - kubectl delete pods --selector app=l3_distributedattackdetectorservice + - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" - kubectl get all + # environment: + # name: test + # url: https://example.com + # kubernetes: + # namespace: test + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + when: manual + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + when: manual -- GitLab