From 7a04088a9013a0ee07c9fac2832ffdb1149745e3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 13 Mar 2024 12:34:00 +0000 Subject: [PATCH 01/73] CI/CD pipeline - end-to-end automated tests: - Integrate OFC'22 TFS deployment --- .gitlab-ci.yml | 8 ++++--- src/tests/.gitlab-ci.yml | 8 +++++-- src/tests/ofc22/.gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 src/tests/ofc22/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 141578287..194e93a6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,10 +17,9 @@ stages: #- dependencies - build - unit_test - #- deploy - #- end2end_test + - end2end_test -# include the individual .gitlab-ci.yml of each micro-service +# include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - local: '/src/monitoring/.gitlab-ci.yml' @@ -45,3 +44,6 @@ include: #- local: '/src/dlt/.gitlab-ci.yml' - local: '/src/load_generator/.gitlab-ci.yml' - local: '/src/bgpls_speaker/.gitlab-ci.yml' + + # This should be last one: end-to-end integration tests + - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index db44b9e4a..d48456b83 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# include the individual .gitlab-ci.yml of each integration test +# include the individual .gitlab-ci.yml of each end-to-end integration test include: - local: '/src/tests/ofc22/.gitlab-ci.yml' - - local: '/src/tests/oeccpsc22/.gitlab-ci.yml' + #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' + #- local: '/src/tests/ecoc22/.gitlab-ci.yml' + #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' + #- local: '/src/tests/ofc23/.gitlab-ci.yml' + #- local: '/src/tests/ofc24/.gitlab-ci.yml' diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml new file mode 100644 index 000000000..5f8f8dd1b --- /dev/null +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +end2end ofc22: + variables: + TEST_NAME: 'ofc22' + stage: end2end_test + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - source src/tests/${TEST_NAME}/deploy_specs.sh + - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/" + - export TFS_SKIP_BUILD="YES" + - export TFS_IMAGE_TAG="latest" + - ./deploy/all.sh + - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + ## Run the tests + #- > + # docker exec -i $IMAGE_NAME bash -c + # "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml -- GitLab From 8d0250aa41c63ae9a1e838df6158e7cea2ba4112 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 13 Mar 2024 18:11:57 +0000 Subject: [PATCH 02/73] Fix CI/CD pipeline - end-to-end automated tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 5f8f8dd1b..263d9fbae 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -21,7 +21,7 @@ end2end ofc22: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/" + - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" - export TFS_SKIP_BUILD="YES" - export TFS_IMAGE_TAG="latest" - ./deploy/all.sh -- GitLab From 2ea3e6795a1b5b1383d5fb0dca357c9671bc0e61 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 14 Mar 2024 11:27:26 +0000 Subject: [PATCH 03/73] Fix CI/CD pipeline - end-to-end automated tests --- deploy/crdb.sh | 3 +++ deploy/qdb.sh | 3 +++ deploy/tfs.sh | 15 +++++++++++++-- src/tests/ofc22/.gitlab-ci.yml | 7 ++++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 5d87adf60..5eae5add2 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -158,8 +158,11 @@ function crdb_undeploy_single() { function crdb_drop_database_single() { echo "Drop database if exists" + kubectl --namespace ${CRDB_NAMESPACE} describe service cockroachdb-public -o yaml CRDB_PORT_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') + echo "CRDB_PORT_SQL=${CRDB_PORT_SQL}" CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@cockroachdb-0:${CRDB_PORT_SQL}/defaultdb?sslmode=require" + echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}" kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" diff --git a/deploy/qdb.sh b/deploy/qdb.sh index e930b5a6c..5e4ee761a 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -160,8 +160,11 @@ function qdb_undeploy() { } function qdb_drop_tables() { + kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o yaml QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}') + echo "QDB_HOST=${QDB_HOST}" QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') + echo "QDB_PORT=${QDB_PORT}" echo "Drop tables, if exist" curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;" diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 19c0d75a0..e3a0f8501 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -299,8 +299,13 @@ for COMPONENT in $TFS_COMPONENTS; do VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else - IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') - VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) + if [ "$TFS_SKIP_BUILD" != "YES" ]; then + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) + else + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$VERSION" | sed 's,//,/,g' | sed 's,http:/,,g') + fi sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" fi @@ -365,6 +370,12 @@ for COMPONENT in $TFS_COMPONENTS; do COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl wait --namespace $TFS_K8S_NAMESPACE \ --for='condition=available' --timeout=90s deployment/${COMPONENT_OBJNAME}service + WAIT_EXIT_CODE=$! + if [[ $WAIT_EXIT_CODE != 0 ]]; then + echo " Failed to deploy '$COMPONENT' component, exiting..." + kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/${COMPONENT_OBJNAME}service + exit $exit_code + fi printf "\n" done diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 263d9fbae..86505017f 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -24,7 +24,12 @@ end2end ofc22: - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" - export TFS_SKIP_BUILD="YES" - export TFS_IMAGE_TAG="latest" - - ./deploy/all.sh + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/qdb.sh + - ./deploy/expose_dashboard.sh + - ./deploy/tfs.sh + - ./deploy/show.sh - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server ## Run the tests -- GitLab From 7f5389d5a4642272c4ae731dae9bac026de19121 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 14 Mar 2024 12:55:09 +0000 Subject: [PATCH 04/73] Fix CI/CD pipeline - end-to-end automated tests --- deploy/crdb.sh | 2 +- deploy/tfs.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 5eae5add2..b26ef3ecf 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -158,7 +158,7 @@ function crdb_undeploy_single() { function crdb_drop_database_single() { echo "Drop database if exists" - kubectl --namespace ${CRDB_NAMESPACE} describe service cockroachdb-public -o yaml + kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml CRDB_PORT_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') echo "CRDB_PORT_SQL=${CRDB_PORT_SQL}" CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@cockroachdb-0:${CRDB_PORT_SQL}/defaultdb?sslmode=require" diff --git a/deploy/tfs.sh b/deploy/tfs.sh index e3a0f8501..aeeda08a3 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -373,7 +373,7 @@ for COMPONENT in $TFS_COMPONENTS; do WAIT_EXIT_CODE=$! if [[ $WAIT_EXIT_CODE != 0 ]]; then echo " Failed to deploy '$COMPONENT' component, exiting..." - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/${COMPONENT_OBJNAME}service + kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/${COMPONENT_OBJNAME}service --all-containers=true exit $exit_code fi printf "\n" -- GitLab From 76d28c7fa08c93d9c24eefe256992f04ccecdafc Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 14 Mar 2024 14:43:55 +0000 Subject: [PATCH 05/73] Fix CI/CD pipeline - end-to-end automated tests --- deploy/crdb.sh | 6 ++++-- deploy/qdb.sh | 5 ++--- deploy/tfs.sh | 6 +++--- src/tests/ofc22/.gitlab-ci.yml | 2 +- src/tests/ofc22/deploy_specs.sh | 18 ++++++++++++++++-- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index b26ef3ecf..89ec72454 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -158,10 +158,12 @@ function crdb_undeploy_single() { function crdb_drop_database_single() { echo "Drop database if exists" - kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml + #kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml + CRDB_HOST_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.clusterIP}') + echo "CRDB_HOST_SQL=${CRDB_HOST_SQL}" CRDB_PORT_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') echo "CRDB_PORT_SQL=${CRDB_PORT_SQL}" - CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@cockroachdb-0:${CRDB_PORT_SQL}/defaultdb?sslmode=require" + CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST_SQL}:${CRDB_PORT_SQL}/defaultdb?sslmode=require" echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}" kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ diff --git a/deploy/qdb.sh b/deploy/qdb.sh index 5e4ee761a..09308cf4c 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -160,13 +160,12 @@ function qdb_undeploy() { } function qdb_drop_tables() { - kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o yaml + echo "Drop tables, if exist" + #kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o yaml QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}') echo "QDB_HOST=${QDB_HOST}" QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') echo "QDB_PORT=${QDB_PORT}" - - echo "Drop tables, if exist" curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;" echo curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;" diff --git a/deploy/tfs.sh b/deploy/tfs.sh index aeeda08a3..fd49c9758 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -370,11 +370,11 @@ for COMPONENT in $TFS_COMPONENTS; do COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl wait --namespace $TFS_K8S_NAMESPACE \ --for='condition=available' --timeout=90s deployment/${COMPONENT_OBJNAME}service - WAIT_EXIT_CODE=$! + WAIT_EXIT_CODE=$? if [[ $WAIT_EXIT_CODE != 0 ]]; then - echo " Failed to deploy '$COMPONENT' component, exiting..." + echo " Failed to deploy '${COMPONENT}' component, exit code '${WAIT_EXIT_CODE}', exiting..." kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/${COMPONENT_OBJNAME}service --all-containers=true - exit $exit_code + exit $WAIT_EXIT_CODE fi printf "\n" done diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 86505017f..54e00041f 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -46,4 +46,4 @@ end2end ofc22: artifacts: when: always reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + junit: src/tests/${TEST_NAME}/report*.xml diff --git a/src/tests/ofc22/deploy_specs.sh b/src/tests/ofc22/deploy_specs.sh index d0696f91d..aad41f33d 100755 --- a/src/tests/ofc22/deploy_specs.sh +++ b/src/tests/ofc22/deploy_specs.sh @@ -26,10 +26,18 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" -# Uncomment to activate ZTP and Policy Manager -#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp policy" +# Uncomment to activate BGP-LS Speaker +#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" + +# Uncomment to activate Optical Controller +#export TFS_COMPONENTS="${TFS_COMPONENTS} opticalcontroller" + +# Uncomment to activate ZTP export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + # Uncomment to activate Optical CyberSecurity #export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" @@ -39,6 +47,12 @@ export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" # Uncomment to activate TE #export TFS_COMPONENTS="${TFS_COMPONENTS} te" +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" -- GitLab From 227b8b001da5ab294dedb0b93d48eea26a01c673 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 14 Mar 2024 16:23:45 +0000 Subject: [PATCH 06/73] CI/CD pipeline: - Deactivating optical attack detector and mitigator tests as they are not stable --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 194e93a6d..5a9e05758 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,8 +28,8 @@ include: - local: '/src/device/.gitlab-ci.yml' - local: '/src/service/.gitlab-ci.yml' - local: '/src/dbscanserving/.gitlab-ci.yml' - - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - - local: '/src/opticalattackdetector/.gitlab-ci.yml' + #- local: '/src/opticalattackmitigator/.gitlab-ci.yml' + #- local: '/src/opticalattackdetector/.gitlab-ci.yml' #- local: '/src/opticalattackmanager/.gitlab-ci.yml' - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' -- GitLab From 9b69d5e3fcac2f7cf6a09ca5bde75b68989a9a3b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 14 Mar 2024 17:26:22 +0000 Subject: [PATCH 07/73] CI/CD pipeline: - Activating optical attack * build stages only - Activating ECOC test --- .gitlab-ci.yml | 6 +- src/opticalattackdetector/.gitlab-ci.yml | 124 +++++++++++----------- src/opticalattackmanager/.gitlab-ci.yml | 78 +++++++------- src/opticalattackmitigator/.gitlab-ci.yml | 70 ++++++------ src/tests/.gitlab-ci.yml | 2 +- src/tests/ecoc22/.gitlab-ci.yml | 50 +++++++++ src/tests/ecoc22/deploy_specs.sh | 22 +++- src/tests/ofc22/.gitlab-ci.yml | 13 +-- 8 files changed, 217 insertions(+), 148 deletions(-) create mode 100644 src/tests/ecoc22/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a9e05758..064cedd15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,9 +28,9 @@ include: - local: '/src/device/.gitlab-ci.yml' - local: '/src/service/.gitlab-ci.yml' - local: '/src/dbscanserving/.gitlab-ci.yml' - #- local: '/src/opticalattackmitigator/.gitlab-ci.yml' - #- local: '/src/opticalattackdetector/.gitlab-ci.yml' - #- local: '/src/opticalattackmanager/.gitlab-ci.yml' + - local: '/src/opticalattackmitigator/.gitlab-ci.yml' + - local: '/src/opticalattackdetector/.gitlab-ci.yml' + - local: '/src/opticalattackmanager/.gitlab-ci.yml' - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' - local: '/src/forecaster/.gitlab-ci.yml' diff --git a/src/opticalattackdetector/.gitlab-ci.yml b/src/opticalattackdetector/.gitlab-ci.yml index d329cd265..9b92c66fc 100644 --- a/src/opticalattackdetector/.gitlab-ci.yml +++ b/src/opticalattackdetector/.gitlab-ci.yml @@ -37,68 +37,68 @@ build opticalattackdetector: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# apply unit test to the opticalattackdetector component -unit_test opticalattackdetector: - variables: - IMAGE_NAME: 'opticalattackdetector' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build opticalattackdetector - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - - if docker container ls | grep redis; then docker rm -f redis; else echo "redis image is not in the system"; fi - - if docker container ls | grep dbscanserving; then docker rm -f dbscanserving; else echo "dbscanserving image is not in the system"; fi - script: - - export REDIS_PASSWORD=$(uuidgen) - - docker pull "redis:7.0-alpine" - - docker run --name redis -d --network=teraflowbridge -p 16379:6379 -e REDIS_PASSWORD=${REDIS_PASSWORD} --rm redis:7.0-alpine redis-server --requirepass ${REDIS_PASSWORD} - - while ! docker logs redis 2>&1 | grep -q 'Ready to accept connections'; do sleep 1; done - - docker logs redis - - REDIS_ADDRESS=$(docker inspect redis --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - docker pull "$CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG" - - docker run --name dbscanserving -d -p 10008:10008 --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG "python -m dbscanserving.service" - - docker logs dbscanserving - - DBSCANSERVING_ADDRESS=$(docker inspect dbscanserving --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - > - docker run --name $IMAGE_NAME -d -p 10006:10006 - -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" - -e REDIS_PASSWORD=${REDIS_PASSWORD} - -e DBSCANSERVINGSERVICE_SERVICE_HOST=${DBSCANSERVING_ADDRESS} - -e CACHINGSERVICE_SERVICE_HOST=${REDIS_ADDRESS} - --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - sleep 5 - - docker ps -a - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml" - - docker logs redis - - docker logs dbscanserving - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage xml -o /home/${IMAGE_NAME}/results/${IMAGE_NAME}_coverage.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME - - docker rm -f redis - - docker rm -f dbscanserving - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +## apply unit test to the opticalattackdetector component +#unit_test opticalattackdetector: +# variables: +# IMAGE_NAME: 'opticalattackdetector' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build opticalattackdetector +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi +# - if docker container ls | grep redis; then docker rm -f redis; else echo "redis image is not in the system"; fi +# - if docker container ls | grep dbscanserving; then docker rm -f dbscanserving; else echo "dbscanserving image is not in the system"; fi +# script: +# - export REDIS_PASSWORD=$(uuidgen) +# - docker pull "redis:7.0-alpine" +# - docker run --name redis -d --network=teraflowbridge -p 16379:6379 -e REDIS_PASSWORD=${REDIS_PASSWORD} --rm redis:7.0-alpine redis-server --requirepass ${REDIS_PASSWORD} +# - while ! docker logs redis 2>&1 | grep -q 'Ready to accept connections'; do sleep 1; done +# - docker logs redis +# - REDIS_ADDRESS=$(docker inspect redis --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") +# - docker pull "$CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG" +# - docker run --name dbscanserving -d -p 10008:10008 --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG "python -m dbscanserving.service" +# - docker logs dbscanserving +# - DBSCANSERVING_ADDRESS=$(docker inspect dbscanserving --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - > +# docker run --name $IMAGE_NAME -d -p 10006:10006 +# -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" +# -e REDIS_PASSWORD=${REDIS_PASSWORD} +# -e DBSCANSERVINGSERVICE_SERVICE_HOST=${DBSCANSERVING_ADDRESS} +# -e CACHINGSERVICE_SERVICE_HOST=${REDIS_ADDRESS} +# --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - sleep 5 +# - docker ps -a +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml" +# - docker logs redis +# - docker logs dbscanserving +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage xml -o /home/${IMAGE_NAME}/results/${IMAGE_NAME}_coverage.xml" +# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker rm -f redis +# - docker rm -f dbscanserving +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml # Deployment of the opticalattackdetector service in Kubernetes Cluster diff --git a/src/opticalattackmanager/.gitlab-ci.yml b/src/opticalattackmanager/.gitlab-ci.yml index 1fe77d315..56d3c9d4d 100644 --- a/src/opticalattackmanager/.gitlab-ci.yml +++ b/src/opticalattackmanager/.gitlab-ci.yml @@ -38,45 +38,45 @@ build opticalattackmanager: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# Apply unit test to the component -unit_test opticalattackmanager: - variables: - IMAGE_NAME: 'opticalattackmanager' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build opticalattackmanager - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 10005:10005 -e LOG_LEVEL=DEBUG -v "$PWD/src/$IMAGE_NAME/tests:/home/teraflow/controller/$IMAGE_NAME/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - docker ps -a - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/teraflow/controller/$IMAGE_NAME/results/${IMAGE_NAME}_report.xml; coverage report --include='${IMAGE_NAME}/*' --show-missing" - - ls -la src/$IMAGE_NAME/tests - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +## Apply unit test to the component +#unit_test opticalattackmanager: +# variables: +# IMAGE_NAME: 'opticalattackmanager' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build opticalattackmanager +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker run --name $IMAGE_NAME -d -p 10005:10005 -e LOG_LEVEL=DEBUG -v "$PWD/src/$IMAGE_NAME/tests:/home/teraflow/controller/$IMAGE_NAME/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - docker ps -a +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/teraflow/controller/$IMAGE_NAME/results/${IMAGE_NAME}_report.xml; coverage report --include='${IMAGE_NAME}/*' --show-missing" +# - ls -la src/$IMAGE_NAME/tests +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml # Deployment of the service in Kubernetes Cluster # deploy opticalattackmanager: diff --git a/src/opticalattackmitigator/.gitlab-ci.yml b/src/opticalattackmitigator/.gitlab-ci.yml index 30eb94b1b..e684b94a7 100644 --- a/src/opticalattackmitigator/.gitlab-ci.yml +++ b/src/opticalattackmitigator/.gitlab-ci.yml @@ -36,41 +36,41 @@ build opticalattackmitigator: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# apply unit test to the opticalattackmitigator component -unit_test opticalattackmitigator: - variables: - IMAGE_NAME: 'opticalattackmitigator' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build opticalattackmitigator - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 10007:10007 -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - sleep 5 - - docker ps -a - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml; coverage xml -o /home/${IMAGE_NAME}/results/${IMAGE_NAME}_coverage.xml; coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +## apply unit test to the opticalattackmitigator component +#unit_test opticalattackmitigator: +# variables: +# IMAGE_NAME: 'opticalattackmitigator' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build opticalattackmitigator +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker run --name $IMAGE_NAME -d -p 10007:10007 -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - sleep 5 +# - docker ps -a +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml; coverage xml -o /home/${IMAGE_NAME}/results/${IMAGE_NAME}_coverage.xml; coverage report --include='${IMAGE_NAME}/*' --show-missing" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml # Deployment of the opticalattackmitigator service in Kubernetes Cluster diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index d48456b83..41b8bb36c 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - #- local: '/src/tests/ecoc22/.gitlab-ci.yml' + - local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' #- local: '/src/tests/ofc23/.gitlab-ci.yml' #- local: '/src/tests/ofc24/.gitlab-ci.yml' diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml new file mode 100644 index 000000000..af0a7669e --- /dev/null +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -0,0 +1,50 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +end2end ecoc22: + variables: + TEST_NAME: 'ecoc22' + stage: end2end_test + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - source src/tests/${TEST_NAME}/deploy_specs.sh + - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" + - export TFS_SKIP_BUILD="YES" + - export TFS_IMAGE_TAG="latest" + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/qdb.sh + - ./deploy/expose_dashboard.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + # Run the tests + - source tfs_runtime_env_vars.sh + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report*.xml diff --git a/src/tests/ecoc22/deploy_specs.sh b/src/tests/ecoc22/deploy_specs.sh index f2a19f773..032a8e2bb 100755 --- a/src/tests/ecoc22/deploy_specs.sh +++ b/src/tests/ecoc22/deploy_specs.sh @@ -26,8 +26,17 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" -# Uncomment to activate ZTP and Policy Manager -#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp policy" +# Uncomment to activate BGP-LS Speaker +#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" + +# Uncomment to activate Optical Controller +#export TFS_COMPONENTS="${TFS_COMPONENTS} opticalcontroller" + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" # Uncomment to activate Optical CyberSecurity #export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" @@ -35,6 +44,15 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate L3 CyberSecurity #export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 54e00041f..4c9c444c8 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -32,11 +32,12 @@ end2end ofc22: - ./deploy/show.sh - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - ## Run the tests - #- > - # docker exec -i $IMAGE_NAME bash -c - # "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - #- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + # Run the tests + - source tfs_runtime_env_vars.sh + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml + - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' @@ -46,4 +47,4 @@ end2end ofc22: artifacts: when: always reports: - junit: src/tests/${TEST_NAME}/report*.xml + junit: ./src/tests/${TEST_NAME}/report*.xml -- GitLab From 42edfa817ef43e01be0bb8d417e36f18fb1d4afd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 08:20:49 +0000 Subject: [PATCH 08/73] CI/CD pipeline: - Deactivating ECOC'22 test - Fixed execution of OFC'22 tests --- src/tests/.gitlab-ci.yml | 2 +- src/tests/ofc22/.gitlab-ci.yml | 47 +++++++++++++++++++---- src/tests/ofc22/Dockerfile | 68 +++++++++++++++++++++++++++++++++ src/tests/ofc22/requirements.in | 15 ++++++++ src/tests/ofc22/run_tests.sh | 8 ++-- 5 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 src/tests/ofc22/Dockerfile create mode 100644 src/tests/ofc22/requirements.in diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index 41b8bb36c..d48456b83 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - - local: '/src/tests/ecoc22/.gitlab-ci.yml' + #- local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' #- local: '/src/tests/ofc23/.gitlab-ci.yml' #- local: '/src/tests/ofc24/.gitlab-ci.yml' diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 4c9c444c8..e40eb58d0 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -13,6 +13,30 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry +build end2end: + variables: + TEST_NAME: 'end2end' + IMAGE_TAG: 'latest' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "${TEST_NAME}:$IMAGE_TAG" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test end2end ofc22: variables: TEST_NAME: 'ofc22' @@ -20,24 +44,33 @@ end2end ofc22: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: + # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" - export TFS_SKIP_BUILD="YES" - export TFS_IMAGE_TAG="latest" + + # Deploy TeraFlowSDN - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/qdb.sh - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh + + # Wait for Context to be subscribed to NATS - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - # Run the tests - - source tfs_runtime_env_vars.sh - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + + # Run end-to-end tests + - docker create --name ${TEST_NAME} -d -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG ./run_tests.sh + - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} + - docker start + #- source tfs_runtime_env_vars.sh + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' @@ -47,4 +80,4 @@ end2end ofc22: artifacts: when: always reports: - junit: ./src/tests/${TEST_NAME}/report*.xml + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile new file mode 100644 index 000000000..2dff67e69 --- /dev/null +++ b/src/tests/ofc22/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/ofc22 +WORKDIR /var/teraflow/tests/ofc22 +COPY src/tests/ofc22/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow/tests/ofc22 +COPY src/tests/ofc22/tests/. ./tests/ +COPY src/tests/ofc22/__init__.py __init__.py +COPY src/tests/ofc22/descriptors_emulated.json descriptors_emulated.json +COPY src/tests/ofc22/run_tests.sh run_tests.sh diff --git a/src/tests/ofc22/requirements.in b/src/tests/ofc22/requirements.in new file mode 100644 index 000000000..30b11e653 --- /dev/null +++ b/src/tests/ofc22/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +requests==2.27.* diff --git a/src/tests/ofc22/run_tests.sh b/src/tests/ofc22/run_tests.sh index fe87a2c54..860c1f96c 100755 --- a/src/tests/ofc22/run_tests.sh +++ b/src/tests/ofc22/run_tests.sh @@ -15,7 +15,7 @@ # Run functional tests source tfs_runtime_env_vars.sh -pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_bootstrap.py -pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_create_service.py -pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_delete_service.py -pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_cleanup.py +pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_bootstrap.py --junitxml=/opt/results/report_bootstrap.xml +pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_create_service.py --junitxml=/opt/results/report_create_service.xml +pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_delete_service.py --junitxml=/opt/results/report_delete_service.xml +pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml -- GitLab From f038429a02f278eaa17d09615b237a9e8371c736 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 08:24:17 +0000 Subject: [PATCH 09/73] CI/CD pipeline: - Fixed build of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index e40eb58d0..5c749b9d1 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -13,9 +13,9 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -build end2end: +build ofc22: variables: - TEST_NAME: 'end2end' + TEST_NAME: 'ofc22' IMAGE_TAG: 'latest' stage: build before_script: -- GitLab From ad3b1da540c5eb36d51386ce4b62d1a130591952 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 09:24:57 +0000 Subject: [PATCH 10/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 5c749b9d1..e2c0c4931 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -44,6 +44,11 @@ end2end ofc22: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: + # Check MicroK8s is ready + - microk8s start + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" @@ -63,7 +68,7 @@ end2end ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server # Run end-to-end tests - - docker create --name ${TEST_NAME} -d -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG ./run_tests.sh + - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG ./run_tests.sh - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start #- source tfs_runtime_env_vars.sh -- GitLab From 9ae8c325f2acc7fa86f9d01ae30f009e1eba2704 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 09:39:12 +0000 Subject: [PATCH 11/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index e2c0c4931..db5ae633e 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -45,7 +45,6 @@ end2end ofc22: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: # Check MicroK8s is ready - - microk8s start - microk8s status --wait-ready - kubectl get pods --all-namespaces -- GitLab From 3c3954b29d2d127940e97dd0cfcc54961cd7e8c2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 10:16:33 +0000 Subject: [PATCH 12/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- deploy/crdb.sh | 19 +++++++++++++------ deploy/qdb.sh | 14 +++++++++++--- src/tests/ofc22/.gitlab-ci.yml | 2 ++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 89ec72454..5b1a77a00 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -158,12 +158,19 @@ function crdb_undeploy_single() { function crdb_drop_database_single() { echo "Drop database if exists" - #kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml - CRDB_HOST_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.clusterIP}') - echo "CRDB_HOST_SQL=${CRDB_HOST_SQL}" - CRDB_PORT_SQL=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') - echo "CRDB_PORT_SQL=${CRDB_PORT_SQL}" - CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST_SQL}:${CRDB_PORT_SQL}/defaultdb?sslmode=require" + + if [[ -z "${GITLAB_CI}" ]]; then + #kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o yaml + CRDB_HOST=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.clusterIP}') + CRDB_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') + else + CRDB_HOST="127.0.0.1" + CRDB_PORT=${CRDB_EXT_PORT_SQL} + fi + + echo "CRDB_HOST=${CRDB_HOST}" + echo "CRDB_PORT=${CRDB_PORT}" + CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST}:${CRDB_PORT}/defaultdb?sslmode=require" echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}" kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ diff --git a/deploy/qdb.sh b/deploy/qdb.sh index 09308cf4c..cca270b44 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -161,11 +161,19 @@ function qdb_undeploy() { function qdb_drop_tables() { echo "Drop tables, if exist" - #kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o yaml - QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}') + + if [[ -z "${GITLAB_CI}" ]]; then + #kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o yaml + QDB_HOST=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.clusterIP}') + QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') + else + QDB_HOST="127.0.0.1" + QDB_PORT=${QDB_EXT_PORT_HTTP} + fi + echo "QDB_HOST=${QDB_HOST}" - QDB_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') echo "QDB_PORT=${QDB_PORT}" + curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;" echo curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;" diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index db5ae633e..4a14c516f 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -40,9 +40,11 @@ build ofc22: end2end ofc22: variables: TEST_NAME: 'ofc22' + IMAGE_TAG: 'latest' stage: end2end_test before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" script: # Check MicroK8s is ready - microk8s status --wait-ready -- GitLab From 996c08ab548fa53117d417d5bc187ad1a500dbd2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 13:49:55 +0000 Subject: [PATCH 13/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 4a14c516f..822c06b48 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -44,8 +44,10 @@ end2end ofc22: stage: end2end_test before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" script: + # Download Docker image to run the test + - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" + # Check MicroK8s is ready - microk8s status --wait-ready - kubectl get pods --all-namespaces -- GitLab From a92c789fe146f161b261599dabd8b2f078bfacf4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 14:40:22 +0000 Subject: [PATCH 14/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 822c06b48..5a5851264 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ofc22: - .gitlab-ci.yml # Deploy TeraFlowSDN and Execute end-2-end test -end2end ofc22: +end2end_test ofc22: variables: TEST_NAME: 'ofc22' IMAGE_TAG: 'latest' @@ -45,8 +45,10 @@ end2end ofc22: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: + - echo "TEST_NAME=${TEST_NAME}" + - echo "IMAGE_TAG=${IMAGE_TAG}" # Download Docker image to run the test - - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:${IMAGE_TAG}" # Check MicroK8s is ready - microk8s status --wait-ready @@ -54,9 +56,11 @@ end2end ofc22: # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" + - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - export TFS_SKIP_BUILD="YES" - - export TFS_IMAGE_TAG="latest" + - export TFS_IMAGE_TAG="${IMAGE_TAG}" + - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + - echo "TFS_IMAGE_TAG=${IMAGE_TAG}" # Deploy TeraFlowSDN - ./deploy/crdb.sh -- GitLab From 8fc8f6cf27649cd856268eb08529b40b183864db Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 15:24:15 +0000 Subject: [PATCH 15/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 5a5851264..ef911c29c 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -16,14 +16,13 @@ build ofc22: variables: TEST_NAME: 'ofc22' - IMAGE_TAG: 'latest' stage: build before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "${TEST_NAME}:$IMAGE_TAG" -f ./src/tests/${TEST_NAME}/Dockerfile . - - docker tag "${TEST_NAME}:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" - - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG" + - docker build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi rules: @@ -40,15 +39,12 @@ build ofc22: end2end_test ofc22: variables: TEST_NAME: 'ofc22' - IMAGE_TAG: 'latest' stage: end2end_test before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - echo "TEST_NAME=${TEST_NAME}" - - echo "IMAGE_TAG=${IMAGE_TAG}" # Download Docker image to run the test - - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:${IMAGE_TAG}" + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready @@ -58,9 +54,8 @@ end2end_test ofc22: - source src/tests/${TEST_NAME}/deploy_specs.sh - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - export TFS_SKIP_BUILD="YES" - - export TFS_IMAGE_TAG="${IMAGE_TAG}" + - export TFS_IMAGE_TAG="latest" - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" - - echo "TFS_IMAGE_TAG=${IMAGE_TAG}" # Deploy TeraFlowSDN - ./deploy/crdb.sh @@ -75,7 +70,7 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server # Run end-to-end tests - - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:$IMAGE_TAG ./run_tests.sh + - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest ./run_tests.sh - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start #- source tfs_runtime_env_vars.sh -- GitLab From 0f242f07a01cf298ae2b3f2879803bb8b0021e1a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 15:27:00 +0000 Subject: [PATCH 16/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 064cedd15..9a454253f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,28 +22,28 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - - local: '/src/monitoring/.gitlab-ci.yml' - - local: '/src/nbi/.gitlab-ci.yml' - - local: '/src/context/.gitlab-ci.yml' - - local: '/src/device/.gitlab-ci.yml' - - local: '/src/service/.gitlab-ci.yml' - - local: '/src/dbscanserving/.gitlab-ci.yml' - - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - - local: '/src/opticalattackdetector/.gitlab-ci.yml' - - local: '/src/opticalattackmanager/.gitlab-ci.yml' - - local: '/src/ztp/.gitlab-ci.yml' - - local: '/src/policy/.gitlab-ci.yml' - - local: '/src/forecaster/.gitlab-ci.yml' + ###- local: '/src/monitoring/.gitlab-ci.yml' + ###- local: '/src/nbi/.gitlab-ci.yml' + ###- local: '/src/context/.gitlab-ci.yml' + ###- local: '/src/device/.gitlab-ci.yml' + ###- local: '/src/service/.gitlab-ci.yml' + ###- local: '/src/dbscanserving/.gitlab-ci.yml' + ###- local: '/src/opticalattackmitigator/.gitlab-ci.yml' + ###- local: '/src/opticalattackdetector/.gitlab-ci.yml' + ###- local: '/src/opticalattackmanager/.gitlab-ci.yml' + ###- local: '/src/ztp/.gitlab-ci.yml' + ###- local: '/src/policy/.gitlab-ci.yml' + ###- local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - - local: '/src/slice/.gitlab-ci.yml' + ###- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - - local: '/src/pathcomp/.gitlab-ci.yml' + ###- local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - - local: '/src/load_generator/.gitlab-ci.yml' - - local: '/src/bgpls_speaker/.gitlab-ci.yml' + ###- local: '/src/load_generator/.gitlab-ci.yml' + ###- local: '/src/bgpls_speaker/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' -- GitLab From 272d3c67e11494cac911889fae81bdee88a9ab61 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 16:08:37 +0000 Subject: [PATCH 17/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index ef911c29c..1e091d454 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -72,7 +72,7 @@ end2end_test ofc22: # Run end-to-end tests - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest ./run_tests.sh - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - - docker start + - docker start ${TEST_NAME} #- source tfs_runtime_env_vars.sh #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml -- GitLab From f449627fa7540b8572e005148943a0f98b4601f9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 16:33:23 +0000 Subject: [PATCH 18/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 4 +++- src/tests/ofc22/Dockerfile | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 1e091d454..0f91997e0 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -70,9 +70,11 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server # Run end-to-end tests - - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest ./run_tests.sh + - if docker container ls | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start ${TEST_NAME} + - docker rm -f ${TEST_NAME} #- source tfs_runtime_env_vars.sh #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 2dff67e69..081b7e58f 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -65,4 +65,16 @@ WORKDIR /var/teraflow/tests/ofc22 COPY src/tests/ofc22/tests/. ./tests/ COPY src/tests/ofc22/__init__.py __init__.py COPY src/tests/ofc22/descriptors_emulated.json descriptors_emulated.json -COPY src/tests/ofc22/run_tests.sh run_tests.sh + +RUN tee ./run_tests.sh < Date: Fri, 15 Mar 2024 16:37:54 +0000 Subject: [PATCH 19/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 0f91997e0..5c5b77c42 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -74,13 +74,13 @@ end2end_test ofc22: - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start ${TEST_NAME} - - docker rm -f ${TEST_NAME} #- source tfs_runtime_env_vars.sh #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml after_script: + - if docker container ls | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: -- GitLab From 43fb94df37ba1db50470193d21bdada2c7db09af Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 16:43:33 +0000 Subject: [PATCH 20/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 5c5b77c42..8bd20891b 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -70,7 +70,7 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server # Run end-to-end tests - - if docker container ls | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start ${TEST_NAME} @@ -80,7 +80,7 @@ end2end_test ofc22: #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml after_script: - - if docker container ls | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: -- GitLab From ed7c7917c46290759aac569d4ce41056d8f66bfd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 17:05:54 +0000 Subject: [PATCH 21/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 8bd20891b..7e780db34 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -74,6 +74,8 @@ end2end_test ofc22: - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} - docker start ${TEST_NAME} + - docker wait ${TEST_NAME} + - docker logs ${TEST_NAME} #- source tfs_runtime_env_vars.sh #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml -- GitLab From be051a0faafc96b876b720eabbc90724e3c3594b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 17:17:40 +0000 Subject: [PATCH 22/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 081b7e58f..ae2e7d8e9 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -61,10 +61,13 @@ RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory -WORKDIR /var/teraflow/tests/ofc22 -COPY src/tests/ofc22/tests/. ./tests/ -COPY src/tests/ofc22/__init__.py __init__.py -COPY src/tests/ofc22/descriptors_emulated.json descriptors_emulated.json +WORKDIR /var/teraflow +COPY src/common/. ./common/ +COPY src/tests/ofc22/tests/. ./tests/ofc22/tests/ +COPY src/__init__.py ./__init__.py +COPY src/tests/__init__.py ./tests/__init__.py +COPY src/tests/ofc22/__init__.py ./tests/ofc22/__init__.py +COPY src/tests/ofc22/descriptors_emulated.json ./tests/ofc22/descriptors_emulated.json RUN tee ./run_tests.sh < Date: Fri, 15 Mar 2024 17:28:02 +0000 Subject: [PATCH 23/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/Dockerfile | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index ae2e7d8e9..0c4bba900 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -62,12 +62,21 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/common/. ./common/ -COPY src/tests/ofc22/tests/. ./tests/ofc22/tests/ COPY src/__init__.py ./__init__.py -COPY src/tests/__init__.py ./tests/__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/tests/*.py ./tests/ COPY src/tests/ofc22/__init__.py ./tests/ofc22/__init__.py COPY src/tests/ofc22/descriptors_emulated.json ./tests/ofc22/descriptors_emulated.json +COPY src/tests/ofc22/tests/. ./tests/ofc22/tests/ +COPY src/tests/tools/. ./tests/tools/ RUN tee ./run_tests.sh < Date: Fri, 15 Mar 2024 17:38:46 +0000 Subject: [PATCH 24/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- src/tests/ofc22/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 7e780db34..dca898563 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -72,7 +72,7 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/tests/${TEST_NAME} + - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ - docker start ${TEST_NAME} - docker wait ${TEST_NAME} - docker logs ${TEST_NAME} diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 0c4bba900..c2a07c687 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -80,7 +80,8 @@ COPY src/tests/tools/. ./tests/tools/ RUN tee ./run_tests.sh < Date: Fri, 15 Mar 2024 17:53:11 +0000 Subject: [PATCH 25/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index c2a07c687..532da0727 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -72,6 +72,12 @@ COPY src/device/__init__.py device/__init__.py COPY src/device/client/. device/client/ COPY src/monitoring/__init__.py monitoring/__init__.py COPY src/monitoring/client/. monitoring/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ COPY src/tests/*.py ./tests/ COPY src/tests/ofc22/__init__.py ./tests/ofc22/__init__.py COPY src/tests/ofc22/descriptors_emulated.json ./tests/ofc22/descriptors_emulated.json -- GitLab From 98173a42aab81ea694743880a40b0fb1f3708eea Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 18:27:29 +0000 Subject: [PATCH 26/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 532da0727..4817bd93a 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -78,6 +78,8 @@ COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ COPY src/service/__init__.py service/__init__.py COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ COPY src/tests/*.py ./tests/ COPY src/tests/ofc22/__init__.py ./tests/ofc22/__init__.py COPY src/tests/ofc22/descriptors_emulated.json ./tests/ofc22/descriptors_emulated.json -- GitLab From a8c155e54574a53221899a113a0736ebb90e5d09 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 18:42:13 +0000 Subject: [PATCH 27/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/tests/test_functional_bootstrap.py | 4 ++-- src/tests/ofc22/tests/test_functional_cleanup.py | 4 ++-- src/tests/ofc22/tests/test_functional_create_service.py | 4 ++-- src/tests/ofc22/tests/test_functional_delete_service.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tests/ofc22/tests/test_functional_bootstrap.py b/src/tests/ofc22/tests/test_functional_bootstrap.py index f14960492..5cc1353ff 100644 --- a/src/tests/ofc22/tests/test_functional_bootstrap.py +++ b/src/tests/ofc22/tests/test_functional_bootstrap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, time +import logging, os, time from common.Constants import DEFAULT_CONTEXT_NAME from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty from common.proto.monitoring_pb2 import KpiDescriptorList @@ -26,7 +26,7 @@ from tests.Fixtures import context_client, device_client, monitoring_client # py LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = 'ofc22/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_bootstrap( diff --git a/src/tests/ofc22/tests/test_functional_cleanup.py b/src/tests/ofc22/tests/test_functional_cleanup.py index 122526840..9eeeb6387 100644 --- a/src/tests/ofc22/tests/test_functional_cleanup.py +++ b/src/tests/ofc22/tests/test_functional_cleanup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging, os from common.Constants import DEFAULT_CONTEXT_NAME from common.proto.context_pb2 import ContextId from common.tools.descriptor.Loader import DescriptorLoader, validate_empty_scenario @@ -24,7 +24,7 @@ from tests.Fixtures import context_client, device_client # pylint: disable=un LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = 'ofc22/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_cleanup( diff --git a/src/tests/ofc22/tests/test_functional_create_service.py b/src/tests/ofc22/tests/test_functional_create_service.py index dd7761f38..5a64825cd 100644 --- a/src/tests/ofc22/tests/test_functional_create_service.py +++ b/src/tests/ofc22/tests/test_functional_create_service.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, random +import logging, os, random from common.Constants import DEFAULT_CONTEXT_NAME from common.proto.context_pb2 import ContextId, Empty, ServiceTypeEnum from common.proto.kpi_sample_types_pb2 import KpiSampleType @@ -29,7 +29,7 @@ from .Objects import WIM_SERVICE_CONNECTION_POINTS, WIM_SERVICE_TYPE LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = 'ofc22/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_service_creation(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name diff --git a/src/tests/ofc22/tests/test_functional_delete_service.py b/src/tests/ofc22/tests/test_functional_delete_service.py index 4fffc115e..649b1abc2 100644 --- a/src/tests/ofc22/tests/test_functional_delete_service.py +++ b/src/tests/ofc22/tests/test_functional_delete_service.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging, os from common.Constants import DEFAULT_CONTEXT_NAME from common.proto.context_pb2 import ContextId, ServiceTypeEnum from common.tools.descriptor.Loader import DescriptorLoader @@ -26,7 +26,7 @@ from .Fixtures import osm_wim # pylint: disable=unused-import LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = 'ofc22/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_service_removal(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name -- GitLab From 6325a79839f71c3f535a06c5e224659779e9f1d9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 19:16:05 +0000 Subject: [PATCH 28/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index dca898563..50c926c2c 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -71,7 +71,7 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - - docker create --name ${TEST_NAME} -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ - docker start ${TEST_NAME} - docker wait ${TEST_NAME} -- GitLab From 895cf8cb15cc9175f5d649af66a76d5af5adc949 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 19:27:11 +0000 Subject: [PATCH 29/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/tests/test_functional_bootstrap.py | 2 +- src/tests/ofc22/tests/test_functional_cleanup.py | 2 +- src/tests/ofc22/tests/test_functional_create_service.py | 2 +- src/tests/ofc22/tests/test_functional_delete_service.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/ofc22/tests/test_functional_bootstrap.py b/src/tests/ofc22/tests/test_functional_bootstrap.py index 5cc1353ff..8b1f98cd1 100644 --- a/src/tests/ofc22/tests/test_functional_bootstrap.py +++ b/src/tests/ofc22/tests/test_functional_bootstrap.py @@ -26,7 +26,7 @@ from tests.Fixtures import context_client, device_client, monitoring_client # py LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_bootstrap( diff --git a/src/tests/ofc22/tests/test_functional_cleanup.py b/src/tests/ofc22/tests/test_functional_cleanup.py index 9eeeb6387..b1396f49a 100644 --- a/src/tests/ofc22/tests/test_functional_cleanup.py +++ b/src/tests/ofc22/tests/test_functional_cleanup.py @@ -24,7 +24,7 @@ from tests.Fixtures import context_client, device_client # pylint: disable=un LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_cleanup( diff --git a/src/tests/ofc22/tests/test_functional_create_service.py b/src/tests/ofc22/tests/test_functional_create_service.py index 5a64825cd..74c74483e 100644 --- a/src/tests/ofc22/tests/test_functional_create_service.py +++ b/src/tests/ofc22/tests/test_functional_create_service.py @@ -29,7 +29,7 @@ from .Objects import WIM_SERVICE_CONNECTION_POINTS, WIM_SERVICE_TYPE LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_service_creation(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name diff --git a/src/tests/ofc22/tests/test_functional_delete_service.py b/src/tests/ofc22/tests/test_functional_delete_service.py index 649b1abc2..daff29064 100644 --- a/src/tests/ofc22/tests/test_functional_delete_service.py +++ b/src/tests/ofc22/tests/test_functional_delete_service.py @@ -26,7 +26,7 @@ from .Fixtures import osm_wim # pylint: disable=unused-import LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.dirname(os.path.abspath(__file__)) + '/descriptors_emulated.json' +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_service_removal(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name -- GitLab From fbbc29e8cfc69606d3327132cb9698c9ac31426e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 19:39:51 +0000 Subject: [PATCH 30/73] CI/CD pipeline: - Fixed execution of OFC'22 tests --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 50c926c2c..fcc11fee8 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -71,7 +71,7 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}/tests:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ - docker start ${TEST_NAME} - docker wait ${TEST_NAME} -- GitLab From d4cf64edffa1012d1d0884f6fc14921a633a76dd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 19:40:20 +0000 Subject: [PATCH 31/73] Service component: - Removed wrongly-placed method call in task executor --- src/service/service/task_scheduler/TaskExecutor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/service/service/task_scheduler/TaskExecutor.py b/src/service/service/task_scheduler/TaskExecutor.py index b9715aae6..537962632 100644 --- a/src/service/service/task_scheduler/TaskExecutor.py +++ b/src/service/service/task_scheduler/TaskExecutor.py @@ -112,7 +112,6 @@ class TaskExecutor: return device def configure_device(self, device : Device) -> None: - self._context_client.SelectOpticalConfig() device_key = get_device_key(device.device_id) self._device_client.ConfigureDevice(device) self._store_grpc_object(CacheableObjectType.DEVICE, device_key, device) -- GitLab From 041f860409e069a11abacdc1d35bb279c6aa4e68 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 21:01:30 +0000 Subject: [PATCH 32/73] Deploy scripts: - Code cleanup --- deploy/crdb.sh | 2 -- deploy/qdb.sh | 3 --- 2 files changed, 5 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 5b1a77a00..b2ff1b03b 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -168,8 +168,6 @@ function crdb_drop_database_single() { CRDB_PORT=${CRDB_EXT_PORT_SQL} fi - echo "CRDB_HOST=${CRDB_HOST}" - echo "CRDB_PORT=${CRDB_PORT}" CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST}:${CRDB_PORT}/defaultdb?sslmode=require" echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}" kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ diff --git a/deploy/qdb.sh b/deploy/qdb.sh index cca270b44..3235c6c82 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -171,9 +171,6 @@ function qdb_drop_tables() { QDB_PORT=${QDB_EXT_PORT_HTTP} fi - echo "QDB_HOST=${QDB_HOST}" - echo "QDB_PORT=${QDB_PORT}" - curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;" echo curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;" -- GitLab From 871f4de78be581f938a98b040fab1049e45e3652 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 21:02:26 +0000 Subject: [PATCH 33/73] ECOC'22 test: - Updated and containerized test - Adapted CI/CD pipeline - Activated CI/CD pipeline --- src/tests/.gitlab-ci.yml | 2 +- src/tests/ecoc22/.gitlab-ci.yml | 62 +++++++++-- src/tests/ecoc22/Dockerfile | 101 ++++++++++++++++++ src/tests/ecoc22/requirements.in | 15 +++ .../ecoc22/tests/test_functional_bootstrap.py | 4 +- .../ecoc22/tests/test_functional_cleanup.py | 4 +- .../tests/test_functional_create_service.py | 4 +- .../tests/test_functional_delete_service.py | 4 +- 8 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 src/tests/ecoc22/Dockerfile create mode 100644 src/tests/ecoc22/requirements.in diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index d48456b83..41b8bb36c 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - #- local: '/src/tests/ecoc22/.gitlab-ci.yml' + - local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' #- local: '/src/tests/ofc23/.gitlab-ci.yml' #- local: '/src/tests/ofc24/.gitlab-ci.yml' diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index af0a7669e..8bf2929e6 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -13,32 +13,76 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -end2end ecoc22: +build ecoc22: + variables: + TEST_NAME: 'ecoc22' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test ecoc22: variables: TEST_NAME: 'ecoc22' stage: end2end_test before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + + # Check MicroK8s is ready + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + + # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="labs.etsi.org:5050/tfs/controller" + - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - export TFS_SKIP_BUILD="YES" - export TFS_IMAGE_TAG="latest" + - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + + # Deploy TeraFlowSDN - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/qdb.sh - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh + + # Wait for Context to be subscribed to NATS - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - # Run the tests - - source tfs_runtime_env_vars.sh - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml - - pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + + # Run end-to-end tests + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ + - docker start ${TEST_NAME} + - docker wait ${TEST_NAME} + - docker logs ${TEST_NAME} + #- source tfs_runtime_env_vars.sh + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml + #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml after_script: + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: @@ -47,4 +91,4 @@ end2end ecoc22: artifacts: when: always reports: - junit: ./src/tests/${TEST_NAME}/report*.xml + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/ecoc22/Dockerfile b/src/tests/ecoc22/Dockerfile new file mode 100644 index 000000000..3ac134a38 --- /dev/null +++ b/src/tests/ecoc22/Dockerfile @@ -0,0 +1,101 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/ecoc22 +WORKDIR /var/teraflow/tests/ecoc22 +COPY src/tests/ecoc22/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/ecoc22/__init__.py ./tests/ecoc22/__init__.py +COPY src/tests/ecoc22/descriptors_emulated.json ./tests/ecoc22/descriptors_emulated.json +COPY src/tests/ecoc22/tests/. ./tests/ecoc22/tests/ +COPY src/tests/tools/. ./tests/tools/ + +RUN tee ./run_tests.sh < Date: Fri, 15 Mar 2024 21:02:44 +0000 Subject: [PATCH 34/73] OFC'22 tests: - Reverted unneeded script changes --- src/tests/ofc22/run_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/ofc22/run_tests.sh b/src/tests/ofc22/run_tests.sh index 860c1f96c..fe87a2c54 100755 --- a/src/tests/ofc22/run_tests.sh +++ b/src/tests/ofc22/run_tests.sh @@ -15,7 +15,7 @@ # Run functional tests source tfs_runtime_env_vars.sh -pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_bootstrap.py --junitxml=/opt/results/report_bootstrap.xml -pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_create_service.py --junitxml=/opt/results/report_create_service.xml -pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_delete_service.py --junitxml=/opt/results/report_delete_service.xml -pytest --verbose --log-level=INFO ./src/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml +pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_bootstrap.py +pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_create_service.py +pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_delete_service.py +pytest --verbose --log-level=INFO src/tests/ofc22/tests/test_functional_cleanup.py -- GitLab From c22e71c54129f192d1aac198a581690f583a014f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 21:03:00 +0000 Subject: [PATCH 35/73] CI/CD pipeline: - Reactivated all components --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a454253f..064cedd15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,28 +22,28 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - ###- local: '/src/monitoring/.gitlab-ci.yml' - ###- local: '/src/nbi/.gitlab-ci.yml' - ###- local: '/src/context/.gitlab-ci.yml' - ###- local: '/src/device/.gitlab-ci.yml' - ###- local: '/src/service/.gitlab-ci.yml' - ###- local: '/src/dbscanserving/.gitlab-ci.yml' - ###- local: '/src/opticalattackmitigator/.gitlab-ci.yml' - ###- local: '/src/opticalattackdetector/.gitlab-ci.yml' - ###- local: '/src/opticalattackmanager/.gitlab-ci.yml' - ###- local: '/src/ztp/.gitlab-ci.yml' - ###- local: '/src/policy/.gitlab-ci.yml' - ###- local: '/src/forecaster/.gitlab-ci.yml' + - local: '/src/monitoring/.gitlab-ci.yml' + - local: '/src/nbi/.gitlab-ci.yml' + - local: '/src/context/.gitlab-ci.yml' + - local: '/src/device/.gitlab-ci.yml' + - local: '/src/service/.gitlab-ci.yml' + - local: '/src/dbscanserving/.gitlab-ci.yml' + - local: '/src/opticalattackmitigator/.gitlab-ci.yml' + - local: '/src/opticalattackdetector/.gitlab-ci.yml' + - local: '/src/opticalattackmanager/.gitlab-ci.yml' + - local: '/src/ztp/.gitlab-ci.yml' + - local: '/src/policy/.gitlab-ci.yml' + - local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - ###- local: '/src/slice/.gitlab-ci.yml' + - local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - ###- local: '/src/pathcomp/.gitlab-ci.yml' + - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - ###- local: '/src/load_generator/.gitlab-ci.yml' - ###- local: '/src/bgpls_speaker/.gitlab-ci.yml' + - local: '/src/load_generator/.gitlab-ci.yml' + - local: '/src/bgpls_speaker/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' -- GitLab From b0a84fc73f79ca999b92881a814f10a9e1532530 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 21:05:55 +0000 Subject: [PATCH 36/73] CI/CD pipeline: - Defined ofc22 and ecoc22 dependencies --- src/tests/ecoc22/.gitlab-ci.yml | 2 ++ src/tests/ofc22/.gitlab-ci.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 8bf2929e6..0aa65e556 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -40,6 +40,8 @@ end2end_test ecoc22: variables: TEST_NAME: 'ecoc22' stage: end2end_test + needs: + - build ecoc22 before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index fcc11fee8..271e398cf 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -40,6 +40,8 @@ end2end_test ofc22: variables: TEST_NAME: 'ofc22' stage: end2end_test + needs: + - build ofc22 before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: -- GitLab From f43ebf7b6d466bc6b8eb53384d4c40796a3a14f9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Mar 2024 21:13:27 +0000 Subject: [PATCH 37/73] CI/CD pipeline: - Reverted ofc22 and ecoc22 dependencies --- src/tests/ecoc22/.gitlab-ci.yml | 5 +++-- src/tests/ofc22/.gitlab-ci.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 0aa65e556..94df23f88 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -40,8 +40,9 @@ end2end_test ecoc22: variables: TEST_NAME: 'ecoc22' stage: end2end_test - needs: - - build ecoc22 + # Disable to force running it after all other tasks + #needs: + # - build ecoc22 before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 271e398cf..65c58d112 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -40,8 +40,9 @@ end2end_test ofc22: variables: TEST_NAME: 'ofc22' stage: end2end_test - needs: - - build ofc22 + # Disable to force running it after all other tasks + #needs: + # - build ofc22 before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: -- GitLab From 1161f9a69189d07bde742af1549a04f74c07135a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 18 Mar 2024 16:19:43 +0000 Subject: [PATCH 38/73] CI/CD pipeline - OFC'22 & ECOC'22: - Fixed drop CRDB database commands - Added DEBUG log dumps in relevant components --- deploy/crdb.sh | 4 ++-- manifests/contextservice.yaml | 2 +- manifests/deviceservice.yaml | 2 +- manifests/monitoringservice.yaml | 2 +- src/tests/ecoc22/.gitlab-ci.yml | 4 ++++ src/tests/ofc22/.gitlab-ci.yml | 4 ++++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index b2ff1b03b..a304e83d1 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -170,7 +170,7 @@ function crdb_drop_database_single() { CRDB_CLIENT_URL="postgresql://${CRDB_USERNAME}:${CRDB_PASSWORD}@${CRDB_HOST}:${CRDB_PORT}/defaultdb?sslmode=require" echo "CRDB_CLIENT_URL=${CRDB_CLIENT_URL}" - kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ + kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" echo @@ -353,7 +353,7 @@ function crdb_undeploy_cluster() { function crdb_drop_database_cluster() { echo "Drop database if exists" - kubectl exec -it --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \ + kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-client-secure -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --host=cockroachdb-public --execute \ "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" echo diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index df06c86b5..5d0e6102c 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -40,7 +40,7 @@ spec: - name: MB_BACKEND value: "nats" - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY value: "FALSE" - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index 77e421f29..7f7885daf 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:2020"] diff --git a/manifests/monitoringservice.yaml b/manifests/monitoringservice.yaml index dbcfa68a0..1a2059395 100644 --- a/manifests/monitoringservice.yaml +++ b/manifests/monitoringservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" envFrom: - secretRef: name: qdb-data diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 94df23f88..fb54c2d5c 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -84,6 +84,10 @@ end2end_test ecoc22: #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server after_script: - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 65c58d112..d7123a952 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -84,6 +84,10 @@ end2end_test ofc22: #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server after_script: - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi -- GitLab From e0c34c1fd3fa07049e2a41f437912b9d1091eb95 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 18 Mar 2024 17:49:25 +0000 Subject: [PATCH 39/73] CI/CD pipeline - ECOC'22: - Disabled check of NATS subscription (not used) --- src/tests/ecoc22/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index fb54c2d5c..791bf4021 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -69,8 +69,8 @@ end2end_test ecoc22: - ./deploy/show.sh # Wait for Context to be subscribed to NATS - - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi -- GitLab From 3565247dd35c30c85f5e187988beb6f358d57d25 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 18 Mar 2024 18:19:52 +0000 Subject: [PATCH 40/73] CI/CD pipeline - OFC'22 & ECOC'22: - Added DEBUG log dumps in relevant components --- manifests/nbiservice.yaml | 2 +- manifests/pathcompservice.yaml | 2 +- manifests/serviceservice.yaml | 2 +- manifests/sliceservice.yaml | 2 +- src/tests/ecoc22/.gitlab-ci.yml | 6 ++++-- src/tests/ofc22/.gitlab-ci.yml | 4 ++++ 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index de97ba364..f5477aeb4 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -37,7 +37,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:9090"] diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 87d907a72..4beadea8f 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: ENABLE_FORECASTER value: "YES" readinessProbe: diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 7d7bdaa4e..3865fd6c0 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:3030"] diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index e7e5c1604..61f5b1d21 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: SLICE_GROUPING value: "DISABLE" envFrom: diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 791bf4021..cde9055a0 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -86,8 +86,10 @@ end2end_test ecoc22: #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server after_script: - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index d7123a952..f78810665 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -86,6 +86,10 @@ end2end_test ofc22: #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server after_script: -- GitLab From 7536c5f8b5ef394d3ce5993da17598603b166fb3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 19 Mar 2024 09:49:44 +0000 Subject: [PATCH 41/73] Gitlab CI/CD pipeline - OFC'22 & ECOC'22: - Disable unneeded components --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 064cedd15..9a454253f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,28 +22,28 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - - local: '/src/monitoring/.gitlab-ci.yml' - - local: '/src/nbi/.gitlab-ci.yml' - - local: '/src/context/.gitlab-ci.yml' - - local: '/src/device/.gitlab-ci.yml' - - local: '/src/service/.gitlab-ci.yml' - - local: '/src/dbscanserving/.gitlab-ci.yml' - - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - - local: '/src/opticalattackdetector/.gitlab-ci.yml' - - local: '/src/opticalattackmanager/.gitlab-ci.yml' - - local: '/src/ztp/.gitlab-ci.yml' - - local: '/src/policy/.gitlab-ci.yml' - - local: '/src/forecaster/.gitlab-ci.yml' + ###- local: '/src/monitoring/.gitlab-ci.yml' + ###- local: '/src/nbi/.gitlab-ci.yml' + ###- local: '/src/context/.gitlab-ci.yml' + ###- local: '/src/device/.gitlab-ci.yml' + ###- local: '/src/service/.gitlab-ci.yml' + ###- local: '/src/dbscanserving/.gitlab-ci.yml' + ###- local: '/src/opticalattackmitigator/.gitlab-ci.yml' + ###- local: '/src/opticalattackdetector/.gitlab-ci.yml' + ###- local: '/src/opticalattackmanager/.gitlab-ci.yml' + ###- local: '/src/ztp/.gitlab-ci.yml' + ###- local: '/src/policy/.gitlab-ci.yml' + ###- local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - - local: '/src/slice/.gitlab-ci.yml' + ###- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - - local: '/src/pathcomp/.gitlab-ci.yml' + ###- local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - - local: '/src/load_generator/.gitlab-ci.yml' - - local: '/src/bgpls_speaker/.gitlab-ci.yml' + ###- local: '/src/load_generator/.gitlab-ci.yml' + ###- local: '/src/bgpls_speaker/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' -- GitLab From 1e8779507f086a5bf1d28387ec4d0cb6c6a826c3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 19 Mar 2024 10:44:12 +0000 Subject: [PATCH 42/73] Service component: - Corrected composition of pathcomp request - Minor cosmetic improvements --- src/service/service/ServiceServiceServicerImpl.py | 7 ++++--- src/service/service/task_scheduler/TaskExecutor.py | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index f65d5b59a..173b49f34 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -245,9 +245,6 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): num_expected_endpoints = num_disjoint_paths * 2 tasks_scheduler = TasksScheduler(self.service_handler_factory) - if len(service_with_uuids.service_endpoint_ids) >= num_expected_endpoints: - pathcomp_request = PathCompRequest() - pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member if service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY: context_id_x = json_context_id(DEFAULT_CONTEXT_NAME) @@ -309,6 +306,10 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): tasks_scheduler.compose_from_pathcompreply( optical_reply, is_delete=False) else: + if len(service_with_uuids.service_endpoint_ids) >= num_expected_endpoints: + pathcomp_request = PathCompRequest() + pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member + if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : pathcomp_request.shortest_path.Clear() # pylint: disable=no-member else: diff --git a/src/service/service/task_scheduler/TaskExecutor.py b/src/service/service/task_scheduler/TaskExecutor.py index 537962632..5c5747970 100644 --- a/src/service/service/task_scheduler/TaskExecutor.py +++ b/src/service/service/task_scheduler/TaskExecutor.py @@ -119,22 +119,22 @@ class TaskExecutor: # New function Andrea for Optical Devices def configure_optical_device(self, device : Device, settings : str, flows : list, is_opticalband : bool): device_key = get_device_key(device.device_id) - myid = OpticalConfigId() - myid.opticalconfig_uuid = device.device_id.device_uuid.uuid - opticalconfig = OpticalConfig() + optical_config_id = OpticalConfigId() + optical_config_id.opticalconfig_uuid = device.device_id.device_uuid.uuid + optical_config = OpticalConfig() setting = settings.value if settings else "" new_config = {} try: - result = self._context_client.SelectOpticalConfig(myid) + result = self._context_client.SelectOpticalConfig(optical_config_id) new_config = json.loads(result.config) if result is not None : new_config["new_config"] = setting new_config["is_opticalband"] = is_opticalband new_config["flow"] = flows result.config = str(new_config) - opticalconfig.CopyFrom(result) - self._device_client.ConfigureOpticalDevice(opticalconfig) + optical_config.CopyFrom(result) + self._device_client.ConfigureOpticalDevice(optical_config) self._store_grpc_object(CacheableObjectType.DEVICE, device_key, device) except Exception as e: LOGGER.info("error in config my config %s",e) -- GitLab From c51bdc9936cebe922c7160172df202c58fd21217 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 19 Mar 2024 10:44:54 +0000 Subject: [PATCH 43/73] Gitlab CI/CD pipeline - OFC'22 & ECOC'22: - Enabled re-build of components in OFC'22 & ECOC'22 tests --- src/tests/ecoc22/.gitlab-ci.yml | 2 +- src/tests/ofc22/.gitlab-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index cde9055a0..109d8f155 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -56,7 +56,7 @@ end2end_test ecoc22: # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="YES" + - export TFS_SKIP_BUILD="" - export TFS_IMAGE_TAG="latest" - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index f78810665..b55948714 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -56,7 +56,7 @@ end2end_test ofc22: # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="YES" + - export TFS_SKIP_BUILD="" - export TFS_IMAGE_TAG="latest" - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" -- GitLab From 818c88cbae6628701c17e13e4225bbc021d99fde Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 19 Mar 2024 13:59:43 +0000 Subject: [PATCH 44/73] Gitlab CI/CD pipeline - OFC'22 & ECOC'22: - Corrected CI/CD pipeline descriptor deploy scripts --- src/tests/ecoc22/.gitlab-ci.yml | 8 ++++---- src/tests/ofc22/.gitlab-ci.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 109d8f155..ead4ceed4 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -55,10 +55,10 @@ end2end_test ecoc22: # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="" - - export TFS_IMAGE_TAG="latest" - - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index b55948714..9598d4eba 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -55,10 +55,10 @@ end2end_test ofc22: # Configure TeraFlowSDN deployment - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="" - - export TFS_IMAGE_TAG="latest" - - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh -- GitLab From ff70a457c11ebbf2fa79f5e94de6fb814ee235b8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 10:28:55 +0000 Subject: [PATCH 45/73] Service component: - Corrected logic for processing a normal service request - Corrected imports of ServicerImpl --- .../service/ServiceServiceServicerImpl.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 173b49f34..b5623885f 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -12,19 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy, grpc, json, logging, random, uuid +import grpc, json, logging, random, uuid from typing import Optional from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.method_wrappers.ServiceExceptions import ( AlreadyExistsException, InvalidArgumentException, NotFoundException, NotImplementedException, OperationFailedException) from common.proto.context_pb2 import ( - Connection, Empty, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum, ConstraintActionEnum) + Connection, ConstraintActionEnum, Empty, Service, ServiceId, ServiceStatusEnum, + ServiceTypeEnum, TopologyId +) from common.proto.pathcomp_pb2 import PathCompRequest from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest from common.proto.service_pb2_grpc import ServiceServiceServicer from common.tools.context_queries.Service import get_service_by_id from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Topology import json_topology_id +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME from context.client.ContextClient import ContextClient from e2e_orchestrator.client.E2EOrchestratorClient import E2EOrchestratorClient from pathcomp.frontend.client.PathCompClient import PathCompClient @@ -33,12 +38,9 @@ from service.client.TEServiceClient import TEServiceClient from .service_handler_api.ServiceHandlerFactory import ServiceHandlerFactory from .task_scheduler.TaskScheduler import TasksScheduler from .tools.GeodesicDistance import gps_distance - -from common.tools.object_factory.Context import json_context_id -from common.tools.object_factory.Topology import json_topology_id -from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME -from common.proto.context_pb2 import Empty, TopologyId -from service.service.tools.OpticalTools import add_lightpath, delete_lightpath, adapt_reply, get_device_name_from_uuid, get_optical_band +from .tools.OpticalTools import ( + add_lightpath, delete_lightpath, adapt_reply, get_device_name_from_uuid, get_optical_band +) LOGGER = logging.getLogger(__name__) @@ -310,19 +312,19 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): pathcomp_request = PathCompRequest() pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member - if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : - pathcomp_request.shortest_path.Clear() # pylint: disable=no-member - else: - pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member + if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : + pathcomp_request.shortest_path.Clear() # pylint: disable=no-member + else: + pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member - pathcomp = PathCompClient() - pathcomp_reply = pathcomp.Compute(pathcomp_request) - pathcomp.close() + pathcomp = PathCompClient() + pathcomp_reply = pathcomp.Compute(pathcomp_request) + pathcomp.close() - # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among - # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be - # executed) to implement the requested create/update operation. - tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) + # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among + # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be + # executed) to implement the requested create/update operation. + tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) tasks_scheduler.execute_all() return service_with_uuids.service_id -- GitLab From 83adf5adb2e3af8ea5719555fca336eb590ef8ad Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 10:40:40 +0000 Subject: [PATCH 46/73] Fixed GitLab CI/CD pipeline - ECOC'22 --- src/tests/ecoc22/.gitlab-ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index ead4ceed4..a1b262131 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -74,16 +74,12 @@ end2end_test ecoc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ - - docker start ${TEST_NAME} - - docker wait ${TEST_NAME} + - > + docker run -it --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - docker logs ${TEST_NAME} - #- source tfs_runtime_env_vars.sh - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend -- GitLab From a6504920af0424feb2a5d6133640cb557c947909 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 11:57:53 +0000 Subject: [PATCH 47/73] Context component: - Deactivated unneeded log messages --- src/context/service/database/Link.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/context/service/database/Link.py b/src/context/service/database/Link.py index e2060c454..ccc22a472 100644 --- a/src/context/service/database/Link.py +++ b/src/context/service/database/Link.py @@ -159,15 +159,15 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) stmt = stmt.returning(TopologyLinkModel.topology_uuid) topology_uuids = session.execute(stmt).fetchall() - LOGGER.warning('RAW topology_uuids={:s}'.format(str(topology_uuids))) + #LOGGER.warning('RAW topology_uuids={:s}'.format(str(topology_uuids))) if len(topology_uuids) > 0: topology_uuids = [topology_uuid[0] for topology_uuid in topology_uuids] - LOGGER.warning('NEW topology_uuids={:s}'.format(str(topology_uuids))) + #LOGGER.warning('NEW topology_uuids={:s}'.format(str(topology_uuids))) query = session.query(TopologyModel) query = query.filter(TopologyModel.topology_uuid.in_(topology_uuids)) link_topologies : List[TopologyModel] = query.all() link_topology_ids = [obj.dump_id() for obj in link_topologies] - LOGGER.warning('link_topology_ids={:s}'.format(str(link_topology_ids))) + #LOGGER.warning('link_topology_ids={:s}'.format(str(link_topology_ids))) return updated or updated_endpoints, link_topology_ids -- GitLab From dd0867d885da8ca3a440a3c8ee76e110cad24288 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 11:58:29 +0000 Subject: [PATCH 48/73] Improved GitLab CI/CD pipeline - ECOC'22: - Added modification of log levels in manifests during deployment --- src/tests/ecoc22/.gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index a1b262131..32949a3ce 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -54,6 +54,12 @@ end2end_test ecoc22: - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="" -- GitLab From 3698661f5d3a743bb5e92594bb30175c4268b42a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:07:25 +0000 Subject: [PATCH 49/73] GitLab CI/CD pipeline: - Disabled OFC'22 while resolving issues --- src/tests/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index 41b8bb36c..6a4441d19 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -14,7 +14,7 @@ # include the individual .gitlab-ci.yml of each end-to-end integration test include: - - local: '/src/tests/ofc22/.gitlab-ci.yml' + #- local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' -- GitLab From 4ae18964c7772a625fdd3c180956fb24712b25c8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:11:07 +0000 Subject: [PATCH 50/73] Fixed GitLab CI/CD pipeline - ECOC'22 --- src/tests/ecoc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 32949a3ce..9f20076bf 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -81,7 +81,7 @@ end2end_test ecoc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -it --name ${TEST_NAME} --network=host + docker run -t --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest -- GitLab From 65444b9062bbcdb635c08b3e7ad56c1c57cdc4c6 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:22:28 +0000 Subject: [PATCH 51/73] Manifests: - Recovered default log levels to INFO - Corrected Formatting of manifest files --- manifests/bgpls_speakerservice.yaml | 60 ++-- manifests/cachingservice.yaml | 50 ++-- manifests/contextservice.yaml | 94 +++---- manifests/dbscanservingservice.yaml | 68 ++--- manifests/deviceservice.yaml | 70 ++--- manifests/dltservice.yaml | 118 ++++---- manifests/e2e_orchestratorservice.yaml | 68 ++--- manifests/forecasterservice.yaml | 86 +++--- manifests/interdomainservice.yaml | 64 ++--- manifests/l3_attackmitigatorservice.yaml | 86 +++--- .../l3_centralizedattackdetectorservice.yaml | 102 +++---- .../l3_distributedattackdetectorservice.yaml | 48 ++-- manifests/load_generatorservice.yaml | 60 ++-- manifests/mock_blockchain.yaml | 50 ++-- manifests/monitoringservice.yaml | 66 ++--- manifests/nbiservice.yaml | 70 ++--- manifests/nginx_ingress_http.yaml | 74 ++--- manifests/opticalattackdetectorservice.yaml | 78 +++--- manifests/opticalattackmanagerservice.yaml | 66 ++--- manifests/opticalattackmitigatorservice.yaml | 68 ++--- manifests/opticalcontrollerservice.yaml | 60 ++-- manifests/pathcompservice.yaml | 132 ++++----- manifests/policyservice.yaml | 12 +- manifests/prometheus.yaml | 31 ++- manifests/servicemonitors.yaml | 256 +++++++++--------- manifests/serviceservice.yaml | 72 ++--- manifests/sliceservice.yaml | 82 +++--- manifests/teservice.yaml | 78 +++--- manifests/webuiservice.yaml | 146 +++++----- manifests/ztpservice.yaml | 12 +- 30 files changed, 1164 insertions(+), 1163 deletions(-) diff --git a/manifests/bgpls_speakerservice.yaml b/manifests/bgpls_speakerservice.yaml index aa985d13e..bda0d4cf5 100644 --- a/manifests/bgpls_speakerservice.yaml +++ b/manifests/bgpls_speakerservice.yaml @@ -28,28 +28,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: localhost:32000/tfs/bgpls_speaker:dev - imagePullPolicy: Always - ports: - - containerPort: 20030 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:20030"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:20030"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: localhost:32000/tfs/bgpls_speaker:dev + imagePullPolicy: Always + ports: + - containerPort: 20030 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:20030"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:20030"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -62,11 +62,11 @@ spec: selector: app: bgpls-speakerservice ports: - - name: grpc - protocol: TCP - port: 20030 - targetPort: 20030 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 20030 + targetPort: 20030 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/cachingservice.yaml b/manifests/cachingservice.yaml index be8fced49..ee6e02e00 100644 --- a/manifests/cachingservice.yaml +++ b/manifests/cachingservice.yaml @@ -27,28 +27,28 @@ spec: app: cachingservice spec: containers: - - name: redis - image: redis:7.0-alpine - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: redis-secrets - key: REDIS_PASSWORD - ports: - - containerPort: 6379 - name: client - command: ["redis-server"] - args: - - --requirepass - - $(REDIS_PASSWORD) - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 512Mi + - name: redis + image: redis:7.0-alpine + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secrets + key: REDIS_PASSWORD + ports: + - containerPort: 6379 + name: client + command: ["redis-server"] + args: + - --requirepass + - $(REDIS_PASSWORD) + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi --- apiVersion: v1 kind: Service @@ -59,6 +59,6 @@ spec: selector: app: cachingservice ports: - - name: redis - port: 6379 - targetPort: 6379 + - name: redis + port: 6379 + targetPort: 6379 diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 5d0e6102c..c7dc59625 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -30,39 +30,39 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/context:latest - imagePullPolicy: Always - ports: - - containerPort: 1010 - - containerPort: 9192 - env: - - name: MB_BACKEND - value: "nats" - - name: LOG_LEVEL - value: "DEBUG" - - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY - value: "FALSE" - - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY - value: "FALSE" - envFrom: - - secretRef: - name: crdb-data - - secretRef: - name: nats-data - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:1010"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:1010"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/context:latest + imagePullPolicy: Always + ports: + - containerPort: 1010 + - containerPort: 9192 + env: + - name: MB_BACKEND + value: "nats" + - name: LOG_LEVEL + value: "INFO" + - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY + value: "FALSE" + - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY + value: "FALSE" + envFrom: + - secretRef: + name: crdb-data + - secretRef: + name: nats-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:1010"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:1010"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -75,14 +75,14 @@ spec: selector: app: contextservice ports: - - name: grpc - protocol: TCP - port: 1010 - targetPort: 1010 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 1010 + targetPort: 1010 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -96,12 +96,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/dbscanservingservice.yaml b/manifests/dbscanservingservice.yaml index b5b8fc437..2bdd1c3fa 100644 --- a/manifests/dbscanservingservice.yaml +++ b/manifests/dbscanservingservice.yaml @@ -27,28 +27,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/dbscanserving:latest - imagePullPolicy: Always - ports: - - containerPort: 10008 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10008"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10008"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/dbscanserving:latest + imagePullPolicy: Always + ports: + - containerPort: 10008 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10008"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10008"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -61,12 +61,12 @@ spec: selector: app: dbscanservingservice ports: - - name: grpc - port: 10008 - targetPort: 10008 - - name: metrics - port: 9192 - targetPort: 9192 + - name: grpc + port: 10008 + targetPort: 10008 + - name: metrics + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -80,12 +80,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index 7f7885daf..fdc3cea02 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -31,33 +31,33 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/device:latest - imagePullPolicy: Always - ports: - - containerPort: 2020 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - startupProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:2020"] - failureThreshold: 30 - periodSeconds: 1 - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:2020"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:2020"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/device:latest + imagePullPolicy: Always + ports: + - containerPort: 2020 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + startupProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:2020"] + failureThreshold: 30 + periodSeconds: 1 + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:2020"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:2020"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -70,11 +70,11 @@ spec: selector: app: deviceservice ports: - - name: grpc - protocol: TCP - port: 2020 - targetPort: 2020 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 2020 + targetPort: 2020 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/dltservice.yaml b/manifests/dltservice.yaml index 5e8f745f7..f905749b4 100644 --- a/manifests/dltservice.yaml +++ b/manifests/dltservice.yaml @@ -27,57 +27,57 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: connector - image: labs.etsi.org:5050/tfs/controller/dlt-connector:latest - imagePullPolicy: Always - ports: - - containerPort: 8080 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - ## for debug purposes - #- name: DLT_GATEWAY_HOST - # value: "mock-blockchain.tfs-bchain.svc.cluster.local" - #- name: DLT_GATEWAY_PORT - # value: "50051" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:8080"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:8080"] - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 512Mi - - name: gateway - image: labs.etsi.org:5050/tfs/controller/dlt-gateway:latest - imagePullPolicy: Always - ports: - - containerPort: 50051 - #readinessProbe: - # httpGet: - # path: /health - # port: 8081 - # initialDelaySeconds: 5 - # timeoutSeconds: 5 - #livenessProbe: - # httpGet: - # path: /health - # port: 8081 - # initialDelaySeconds: 5 - # timeoutSeconds: 5 - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: connector + image: labs.etsi.org:5050/tfs/controller/dlt-connector:latest + imagePullPolicy: Always + ports: + - containerPort: 8080 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + ## for debug purposes + #- name: DLT_GATEWAY_HOST + # value: "mock-blockchain.tfs-bchain.svc.cluster.local" + #- name: DLT_GATEWAY_PORT + # value: "50051" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi + - name: gateway + image: labs.etsi.org:5050/tfs/controller/dlt-gateway:latest + imagePullPolicy: Always + ports: + - containerPort: 50051 + #readinessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + #livenessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -90,11 +90,11 @@ spec: selector: app: dltservice ports: - - name: grpc - protocol: TCP - port: 8080 - targetPort: 8080 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 8080 + targetPort: 8080 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/e2e_orchestratorservice.yaml b/manifests/e2e_orchestratorservice.yaml index 899e17fff..90d377711 100644 --- a/manifests/e2e_orchestratorservice.yaml +++ b/manifests/e2e_orchestratorservice.yaml @@ -27,28 +27,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/e2e_orchestrator:latest - imagePullPolicy: Always - ports: - - containerPort: 10050 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10050"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10050"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/e2e_orchestrator:latest + imagePullPolicy: Always + ports: + - containerPort: 10050 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10050"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10050"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -61,12 +61,12 @@ spec: selector: app: e2e-orchestratorservice ports: - - name: grpc - port: 10050 - targetPort: 10050 - - name: metrics - port: 9192 - targetPort: 9192 + - name: grpc + port: 10050 + targetPort: 10050 + - name: metrics + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -80,12 +80,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/forecasterservice.yaml b/manifests/forecasterservice.yaml index 55d4add88..d1136f617 100644 --- a/manifests/forecasterservice.yaml +++ b/manifests/forecasterservice.yaml @@ -28,35 +28,35 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/forecaster:latest - imagePullPolicy: Always - ports: - - containerPort: 10040 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - - name: FORECAST_TO_HISTORY_RATIO - value: "10" - startupProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10040"] - failureThreshold: 30 - periodSeconds: 1 - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10040"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10040"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/forecaster:latest + imagePullPolicy: Always + ports: + - containerPort: 10040 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: FORECAST_TO_HISTORY_RATIO + value: "10" + startupProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10040"] + failureThreshold: 30 + periodSeconds: 1 + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10040"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10040"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -69,14 +69,14 @@ spec: selector: app: forecasterservice ports: - - name: grpc - protocol: TCP - port: 10040 - targetPort: 10040 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 10040 + targetPort: 10040 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -90,12 +90,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/interdomainservice.yaml b/manifests/interdomainservice.yaml index a23583402..ad9be3a3e 100644 --- a/manifests/interdomainservice.yaml +++ b/manifests/interdomainservice.yaml @@ -27,30 +27,30 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/interdomain:latest - imagePullPolicy: Always - ports: - - containerPort: 10010 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - - name: TOPOLOGY_ABSTRACTOR - value: "DISABLE" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10010"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10010"] - resources: - requests: - cpu: 250m - memory: 64Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/interdomain:latest + imagePullPolicy: Always + ports: + - containerPort: 10010 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: TOPOLOGY_ABSTRACTOR + value: "DISABLE" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10010"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10010"] + resources: + requests: + cpu: 250m + memory: 64Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -63,11 +63,11 @@ spec: selector: app: interdomainservice ports: - - name: grpc - protocol: TCP - port: 10010 - targetPort: 10010 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 10010 + targetPort: 10010 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/l3_attackmitigatorservice.yaml b/manifests/l3_attackmitigatorservice.yaml index ee97d2c92..973b805bd 100644 --- a/manifests/l3_attackmitigatorservice.yaml +++ b/manifests/l3_attackmitigatorservice.yaml @@ -27,28 +27,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/l3_attackmitigator:latest - imagePullPolicy: Always - ports: - - containerPort: 10002 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10002"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10002"] - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/l3_attackmitigator:latest + imagePullPolicy: Always + ports: + - containerPort: 10002 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10002"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10002"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -61,13 +61,13 @@ spec: selector: app: l3-attackmitigatorservice ports: - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 - - name: grpc - port: 10002 - targetPort: 10002 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 + - name: grpc + port: 10002 + targetPort: 10002 --- apiVersion: autoscaling/v2 @@ -82,12 +82,12 @@ spec: minReplicas: 1 maxReplicas: 10 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 behavior: scaleDown: stabilizationWindowSeconds: 120 @@ -100,9 +100,9 @@ metadata: labels: app: l3-attackmitigatorservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -112,11 +112,11 @@ spec: app: l3-attackmitigatorservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running diff --git a/manifests/l3_centralizedattackdetectorservice.yaml b/manifests/l3_centralizedattackdetectorservice.yaml index 8a3be69b6..98c5f9b41 100644 --- a/manifests/l3_centralizedattackdetectorservice.yaml +++ b/manifests/l3_centralizedattackdetectorservice.yaml @@ -27,36 +27,36 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/l3_centralizedattackdetector:latest - imagePullPolicy: Always - ports: - - containerPort: 10001 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - - name: BATCH_SIZE - value: "256" - - name: CAD_CLASSIFICATION_THRESHOLD - value: "0.5" - - name: MONITORED_KPIS_TIME_INTERVAL_AGG - value: "60" - - name: TEST_ML_MODEL - value: "0" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10001"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10001"] - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/l3_centralizedattackdetector:latest + imagePullPolicy: Always + ports: + - containerPort: 10001 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: BATCH_SIZE + value: "256" + - name: CAD_CLASSIFICATION_THRESHOLD + value: "0.5" + - name: MONITORED_KPIS_TIME_INTERVAL_AGG + value: "60" + - name: TEST_ML_MODEL + value: "0" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10001"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10001"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -69,13 +69,13 @@ spec: selector: app: l3-centralizedattackdetectorservice ports: - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 - - name: grpc - port: 10001 - targetPort: 10001 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 + - name: grpc + port: 10001 + targetPort: 10001 --- apiVersion: autoscaling/v2 @@ -90,12 +90,12 @@ spec: minReplicas: 1 maxReplicas: 10 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 behavior: scaleDown: stabilizationWindowSeconds: 120 @@ -107,9 +107,9 @@ metadata: labels: app: l3-centralizedattackdetectorservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -119,11 +119,11 @@ spec: app: l3-centralizedattackdetectorservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running diff --git a/manifests/l3_distributedattackdetectorservice.yaml b/manifests/l3_distributedattackdetectorservice.yaml index b363c1d5c..bf72b5cd0 100644 --- a/manifests/l3_distributedattackdetectorservice.yaml +++ b/manifests/l3_distributedattackdetectorservice.yaml @@ -27,27 +27,27 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/l3_distributedattackdetector:latest - imagePullPolicy: Always - ports: - - containerPort: 10000 - env: - - name: LOG_LEVEL - value: "DEBUG" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10000"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10000"] - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/l3_distributedattackdetector:latest + imagePullPolicy: Always + ports: + - containerPort: 10000 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10000"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10000"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -58,6 +58,6 @@ spec: selector: app: l3-distributedattackdetectorservice ports: - - name: grpc - port: 10000 - targetPort: 10000 + - name: grpc + port: 10000 + targetPort: 10000 diff --git a/manifests/load_generatorservice.yaml b/manifests/load_generatorservice.yaml index 7cc6f1912..bda284ebd 100644 --- a/manifests/load_generatorservice.yaml +++ b/manifests/load_generatorservice.yaml @@ -28,28 +28,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/load_generator:latest - imagePullPolicy: Always - ports: - - containerPort: 50052 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:50052"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:50052"] - resources: - requests: - cpu: 256m - memory: 64Mi - limits: - cpu: 512m - memory: 128Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/load_generator:latest + imagePullPolicy: Always + ports: + - containerPort: 50052 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50052"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50052"] + resources: + requests: + cpu: 256m + memory: 64Mi + limits: + cpu: 512m + memory: 128Mi --- apiVersion: v1 kind: Service @@ -62,11 +62,11 @@ spec: selector: app: load-generatorservice ports: - - name: grpc - protocol: TCP - port: 50052 - targetPort: 50052 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 50052 + targetPort: 50052 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/mock_blockchain.yaml b/manifests/mock_blockchain.yaml index 1093610f8..806432be6 100644 --- a/manifests/mock_blockchain.yaml +++ b/manifests/mock_blockchain.yaml @@ -27,27 +27,27 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/mock_blockchain:latest - imagePullPolicy: Always - ports: - - containerPort: 50051 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:50051"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:50051"] - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/mock_blockchain:latest + imagePullPolicy: Always + ports: + - containerPort: 50051 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -58,7 +58,7 @@ spec: selector: app: mock-blockchain ports: - - name: grpc - protocol: TCP - port: 50051 - targetPort: 50051 + - name: grpc + protocol: TCP + port: 50051 + targetPort: 50051 diff --git a/manifests/monitoringservice.yaml b/manifests/monitoringservice.yaml index 1a2059395..3a4d43cd9 100644 --- a/manifests/monitoringservice.yaml +++ b/manifests/monitoringservice.yaml @@ -28,31 +28,31 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/monitoring:latest - imagePullPolicy: Always - ports: - - containerPort: 7070 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - envFrom: - - secretRef: - name: qdb-data - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:7070"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:7070"] - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/monitoring:latest + imagePullPolicy: Always + ports: + - containerPort: 7070 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: qdb-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:7070"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:7070"] + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -65,11 +65,11 @@ spec: selector: app: monitoringservice ports: - - name: grpc - protocol: TCP - port: 7070 - targetPort: 7070 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 7070 + targetPort: 7070 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index f5477aeb4..0a3bd1ea6 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -28,29 +28,29 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/nbi:latest - imagePullPolicy: Always - ports: - - containerPort: 8080 - - containerPort: 9090 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:9090"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:9090"] - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 512Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/nbi:latest + imagePullPolicy: Always + ports: + - containerPort: 8080 + - containerPort: 9090 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:9090"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:9090"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi --- apiVersion: v1 kind: Service @@ -63,15 +63,15 @@ spec: selector: app: nbiservice ports: - - name: http - protocol: TCP - port: 8080 - targetPort: 8080 - - name: grpc - protocol: TCP - port: 9090 - targetPort: 9090 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: http + protocol: TCP + port: 8080 + targetPort: 8080 + - name: grpc + protocol: TCP + port: 9090 + targetPort: 9090 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/nginx_ingress_http.yaml b/manifests/nginx_ingress_http.yaml index e8e8a80e4..91440fb7a 100644 --- a/manifests/nginx_ingress_http.yaml +++ b/manifests/nginx_ingress_http.yaml @@ -20,40 +20,40 @@ metadata: nginx.ingress.kubernetes.io/rewrite-target: /$2 spec: rules: - - http: - paths: - - path: /webui(/|$)(.*) - pathType: Prefix - backend: - service: - name: webuiservice - port: - number: 8004 - - path: /grafana(/|$)(.*) - pathType: Prefix - backend: - service: - name: webuiservice - port: - number: 3000 - - path: /()(restconf/.*) - pathType: Prefix - backend: - service: - name: nbiservice - port: - number: 8080 - - path: /()(debug-api/.*) - pathType: Prefix - backend: - service: - name: nbiservice - port: - number: 8080 - - path: /()(bmw/.*) - pathType: Prefix - backend: - service: - name: nbiservice - port: - number: 8080 + - http: + paths: + - path: /webui(/|$)(.*) + pathType: Prefix + backend: + service: + name: webuiservice + port: + number: 8004 + - path: /grafana(/|$)(.*) + pathType: Prefix + backend: + service: + name: webuiservice + port: + number: 3000 + - path: /()(restconf/.*) + pathType: Prefix + backend: + service: + name: nbiservice + port: + number: 8080 + - path: /()(debug-api/.*) + pathType: Prefix + backend: + service: + name: nbiservice + port: + number: 8080 + - path: /()(bmw/.*) + pathType: Prefix + backend: + service: + name: nbiservice + port: + number: 8080 diff --git a/manifests/opticalattackdetectorservice.yaml b/manifests/opticalattackdetectorservice.yaml index 197c23dd2..11fd62b61 100644 --- a/manifests/opticalattackdetectorservice.yaml +++ b/manifests/opticalattackdetectorservice.yaml @@ -27,33 +27,33 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/opticalattackdetector:latest - imagePullPolicy: Always - ports: - - containerPort: 10006 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: redis-secrets - key: REDIS_PASSWORD - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10006"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10006"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/opticalattackdetector:latest + imagePullPolicy: Always + ports: + - containerPort: 10006 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secrets + key: REDIS_PASSWORD + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10006"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10006"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -66,12 +66,12 @@ spec: selector: app: opticalattackdetectorservice ports: - - name: grpc - port: 10006 - targetPort: 10006 - - name: metrics - port: 9192 - targetPort: 9192 + - name: grpc + port: 10006 + targetPort: 10006 + - name: metrics + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -85,12 +85,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/opticalattackmanagerservice.yaml b/manifests/opticalattackmanagerservice.yaml index f9838bcbb..4e01d76ac 100644 --- a/manifests/opticalattackmanagerservice.yaml +++ b/manifests/opticalattackmanagerservice.yaml @@ -28,33 +28,33 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/opticalattackmanager:latest - imagePullPolicy: Always - ports: - - containerPort: 10005 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - - name: MONITORING_INTERVAL - value: "30" - - name: OPTICALATTACKMANAGERSERVICE_LOOP_MIN_WORKERS - value: "2" # remember to align this with the resource limits - - name: OPTICALATTACKMANAGERSERVICE_LOOP_MAX_WORKERS - value: "10" # remember to align this with the resource limits - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: redis-secrets - key: REDIS_PASSWORD - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 10000m - memory: 10240Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/opticalattackmanager:latest + imagePullPolicy: Always + ports: + - containerPort: 10005 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: MONITORING_INTERVAL + value: "30" + - name: OPTICALATTACKMANAGERSERVICE_LOOP_MIN_WORKERS + value: "2" # remember to align this with the resource limits + - name: OPTICALATTACKMANAGERSERVICE_LOOP_MAX_WORKERS + value: "10" # remember to align this with the resource limits + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secrets + key: REDIS_PASSWORD + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 10000m + memory: 10240Mi --- apiVersion: v1 kind: Service @@ -67,9 +67,9 @@ spec: selector: app: opticalattackmanagerservice ports: - - name: grpc - port: 10005 - targetPort: 10005 - - name: metrics - port: 9192 - targetPort: 9192 + - name: grpc + port: 10005 + targetPort: 10005 + - name: metrics + port: 9192 + targetPort: 9192 diff --git a/manifests/opticalattackmitigatorservice.yaml b/manifests/opticalattackmitigatorservice.yaml index 4d148b347..255e0fd86 100644 --- a/manifests/opticalattackmitigatorservice.yaml +++ b/manifests/opticalattackmitigatorservice.yaml @@ -27,28 +27,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/opticalattackmitigator:latest - imagePullPolicy: Always - ports: - - containerPort: 10007 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10007"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10007"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/opticalattackmitigator:latest + imagePullPolicy: Always + ports: + - containerPort: 10007 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10007"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10007"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -61,12 +61,12 @@ spec: selector: app: opticalattackmitigatorservice ports: - - name: grpc - port: 10007 - targetPort: 10007 - - name: metrics - port: 9192 - targetPort: 9192 + - name: grpc + port: 10007 + targetPort: 10007 + - name: metrics + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -80,12 +80,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/opticalcontrollerservice.yaml b/manifests/opticalcontrollerservice.yaml index 4b677ee4f..f2351720c 100644 --- a/manifests/opticalcontrollerservice.yaml +++ b/manifests/opticalcontrollerservice.yaml @@ -28,28 +28,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: localhost:32000/tfs/opticalcontroller:dev - imagePullPolicy: Never - ports: - - containerPort: 10060 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "INFO" - #readinessProbe: - # exec: - # command: ["/bin/grpc_health_probe", "-addr=:10060"] - #livenessProbe: - # exec: - # command: ["/bin/grpc_health_probe", "-addr=:10060"] - resources: - requests: - cpu: 500m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: localhost:32000/tfs/opticalcontroller:dev + imagePullPolicy: Never + ports: + - containerPort: 10060 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + #readinessProbe: + # exec: + # command: ["/bin/grpc_health_probe", "-addr=:10060"] + #livenessProbe: + # exec: + # command: ["/bin/grpc_health_probe", "-addr=:10060"] + resources: + requests: + cpu: 500m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -62,11 +62,11 @@ spec: selector: app: opticalcontrollerservice ports: - - name: grpc - protocol: TCP - port: 10060 - targetPort: 10060 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 10060 + targetPort: 10060 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 4beadea8f..f017e6940 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -28,54 +28,54 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: frontend - image: labs.etsi.org:5050/tfs/controller/pathcomp-frontend:latest - imagePullPolicy: Always - ports: - - containerPort: 10020 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - - name: ENABLE_FORECASTER - value: "YES" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10020"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:10020"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi - - name: backend - image: labs.etsi.org:5050/tfs/controller/pathcomp-backend:latest - imagePullPolicy: Always - ports: - - containerPort: 8081 - #readinessProbe: - # httpGet: - # path: /health - # port: 8081 - # initialDelaySeconds: 5 - # timeoutSeconds: 5 - #livenessProbe: - # httpGet: - # path: /health - # port: 8081 - # initialDelaySeconds: 5 - # timeoutSeconds: 5 - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: frontend + image: labs.etsi.org:5050/tfs/controller/pathcomp-frontend:latest + imagePullPolicy: Always + ports: + - containerPort: 10020 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: ENABLE_FORECASTER + value: "NO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10020"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10020"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi + - name: backend + image: labs.etsi.org:5050/tfs/controller/pathcomp-backend:latest + imagePullPolicy: Always + ports: + - containerPort: 8081 + #readinessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + #livenessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -88,18 +88,18 @@ spec: selector: app: pathcompservice ports: - - name: grpc - protocol: TCP - port: 10020 - targetPort: 10020 - - name: http - protocol: TCP - port: 8081 - targetPort: 8081 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 10020 + targetPort: 10020 + - name: http + protocol: TCP + port: 8081 + targetPort: 8081 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -113,12 +113,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/policyservice.yaml b/manifests/policyservice.yaml index b34331724..b93eeda03 100644 --- a/manifests/policyservice.yaml +++ b/manifests/policyservice.yaml @@ -121,9 +121,9 @@ spec: minReplicas: 1 maxReplicas: 10 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 diff --git a/manifests/prometheus.yaml b/manifests/prometheus.yaml index 43a766b6e..ec1ffb0c5 100644 --- a/manifests/prometheus.yaml +++ b/manifests/prometheus.yaml @@ -60,18 +60,18 @@ spec: spec: #serviceAccountName: prometheus containers: - - name: prometheus - image: prom/prometheus:v2.28.1 - ports: - - containerPort: 9090 - volumeMounts: - - name: prometheus-config-volume - mountPath: /etc/prometheus/prometheus.yml - subPath: prometheus.yml + - name: prometheus + image: prom/prometheus:v2.28.1 + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/prometheus.yml + subPath: prometheus.yml volumes: - - name: prometheus-config-volume - configMap: - name: prometheus-config + - name: prometheus-config-volume + configMap: + name: prometheus-config restartPolicy: Always --- apiVersion: v1 @@ -85,8 +85,9 @@ spec: selector: app: prometheus ports: - - name: http - protocol: TCP - port: 9090 - targetPort: 9090 + - name: http + protocol: TCP + port: 9090 + targetPort: 9090 --- + diff --git a/manifests/servicemonitors.yaml b/manifests/servicemonitors.yaml index 1f7f3322d..ccfe774b3 100644 --- a/manifests/servicemonitors.yaml +++ b/manifests/servicemonitors.yaml @@ -20,9 +20,9 @@ metadata: labels: app: contextservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -32,14 +32,14 @@ spec: app: contextservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -49,9 +49,9 @@ metadata: labels: app: deviceservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -61,14 +61,14 @@ spec: app: deviceservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -78,9 +78,9 @@ metadata: labels: app: serviceservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -90,14 +90,14 @@ spec: app: serviceservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -107,9 +107,9 @@ metadata: labels: app: sliceservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -119,14 +119,14 @@ spec: app: sliceservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -136,9 +136,9 @@ metadata: labels: app: pathcompservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -148,14 +148,14 @@ spec: app: pathcompservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -165,9 +165,9 @@ metadata: labels: app: monitoringservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -177,14 +177,14 @@ spec: app: monitoringservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -194,9 +194,9 @@ metadata: labels: app: dltservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -206,14 +206,14 @@ spec: app: dltservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -223,9 +223,9 @@ metadata: labels: app: interdomainservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -235,14 +235,14 @@ spec: app: interdomainservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -252,9 +252,9 @@ metadata: labels: app: policyservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -264,14 +264,14 @@ spec: app: policyservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /q/metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /q/metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -281,9 +281,9 @@ metadata: labels: app: ztpservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -293,14 +293,14 @@ spec: app: ztpservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /q/metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /q/metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -310,9 +310,9 @@ metadata: labels: app: nbiservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -322,14 +322,14 @@ spec: app: nbiservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -339,9 +339,9 @@ metadata: labels: app: load-generatorservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -351,14 +351,14 @@ spec: app: load-generatorservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -368,9 +368,9 @@ metadata: labels: app: dbscanservingservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -380,14 +380,14 @@ spec: app: dbscanservingservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -397,9 +397,9 @@ metadata: labels: app: opticalattackmitigatorservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -409,14 +409,14 @@ spec: app: opticalattackmitigatorservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -426,9 +426,9 @@ metadata: labels: app: opticalattackdetectorservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -438,14 +438,14 @@ spec: app: opticalattackdetectorservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -455,9 +455,9 @@ metadata: labels: app: opticalattackmanagerservice #release: prometheus - #release: prom # name of the release - # ( VERY IMPORTANT: You need to know the correct release name by viewing - # the servicemonitor of Prometheus itself: Without the correct name, + #release: prom # name of the release + # ( VERY IMPORTANT: You need to know the correct release name by viewing + # the servicemonitor of Prometheus itself: Without the correct name, # Prometheus cannot identify the metrics of the Flask app as the target.) spec: selector: @@ -467,11 +467,11 @@ spec: app: opticalattackmanagerservice # same as above #release: prometheus # same as above endpoints: - - port: metrics # named port in target app - scheme: http - path: /metrics # path to scrape - interval: 5s # scrape interval + - port: metrics # named port in target app + scheme: http + path: /metrics # path to scrape + interval: 5s # scrape interval namespaceSelector: any: false matchNames: - - tfs # namespace where the app is running + - tfs # namespace where the app is running diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 3865fd6c0..2fb7ebb87 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -28,28 +28,28 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/service:latest - imagePullPolicy: Always - ports: - - containerPort: 3030 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:3030"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:3030"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/service:latest + imagePullPolicy: Always + ports: + - containerPort: 3030 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:3030"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:3030"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -62,14 +62,14 @@ spec: selector: app: serviceservice ports: - - name: grpc - protocol: TCP - port: 3030 - targetPort: 3030 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 3030 + targetPort: 3030 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -83,12 +83,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index 61f5b1d21..0daa8e70f 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -28,33 +28,33 @@ spec: spec: terminationGracePeriodSeconds: 5 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/slice:latest - imagePullPolicy: Always - ports: - - containerPort: 4040 - - containerPort: 9192 - env: - - name: LOG_LEVEL - value: "DEBUG" - - name: SLICE_GROUPING - value: "DISABLE" - envFrom: - - secretRef: - name: qdb-data - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:4040"] - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:4040"] - resources: - requests: - cpu: 250m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/slice:latest + imagePullPolicy: Always + ports: + - containerPort: 4040 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: SLICE_GROUPING + value: "DISABLE" + envFrom: + - secretRef: + name: qdb-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:4040"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:4040"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -67,14 +67,14 @@ spec: selector: app: sliceservice ports: - - name: grpc - protocol: TCP - port: 4040 - targetPort: 4040 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 4040 + targetPort: 4040 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -88,12 +88,12 @@ spec: minReplicas: 1 maxReplicas: 20 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 #behavior: # scaleDown: # stabilizationWindowSeconds: 30 diff --git a/manifests/teservice.yaml b/manifests/teservice.yaml index 15f1619df..ec8f2e3d6 100644 --- a/manifests/teservice.yaml +++ b/manifests/teservice.yaml @@ -30,37 +30,37 @@ spec: terminationGracePeriodSeconds: 5 shareProcessNamespace: true containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/te:latest - imagePullPolicy: Always - ports: - - containerPort: 10030 - env: - - name: ERLANG_LOGGER_LEVEL - value: "debug" - - name: ERLANG_COOKIE - value: "tfte-unsafe-cookie" - - name: ERLANG_NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: ERLANG_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - readinessProbe: - exec: - command: ["/tfte/bin/tfte", "status"] - livenessProbe: - exec: - command: ["/tfte/bin/tfte", "status"] - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/te:latest + imagePullPolicy: Always + ports: + - containerPort: 10030 + env: + - name: ERLANG_LOGGER_LEVEL + value: "debug" + - name: ERLANG_COOKIE + value: "tfte-unsafe-cookie" + - name: ERLANG_NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ERLANG_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + readinessProbe: + exec: + command: ["/tfte/bin/tfte", "status"] + livenessProbe: + exec: + command: ["/tfte/bin/tfte", "status"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -71,11 +71,11 @@ spec: selector: app: teservice ports: - - name: grpc - protocol: TCP - port: 10030 - targetPort: 10030 - - name: pcep - protocol: TCP - port: 4189 - targetPort: 4189 + - name: grpc + protocol: TCP + port: 10030 + targetPort: 10030 + - name: pcep + protocol: TCP + port: 4189 + targetPort: 4189 diff --git a/manifests/webuiservice.yaml b/manifests/webuiservice.yaml index 43caa9f04..bb2573c45 100644 --- a/manifests/webuiservice.yaml +++ b/manifests/webuiservice.yaml @@ -32,73 +32,73 @@ spec: supplementalGroups: - 0 containers: - - name: server - image: labs.etsi.org:5050/tfs/controller/webui:latest - imagePullPolicy: Always - ports: - - containerPort: 8004 - env: - - name: LOG_LEVEL - value: "INFO" - - name: WEBUISERVICE_SERVICE_BASEURL_HTTP - value: "/webui/" - readinessProbe: - httpGet: - path: /healthz/ready - port: 8004 - initialDelaySeconds: 5 - timeoutSeconds: 1 - livenessProbe: - httpGet: - path: /healthz/live - port: 8004 - initialDelaySeconds: 5 - timeoutSeconds: 1 - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 1000m - memory: 1024Mi - - name: grafana - image: grafana/grafana:8.5.22 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3000 - name: http-grafana - protocol: TCP - env: - - name: GF_SERVER_ROOT_URL - value: "http://0.0.0.0:3000/grafana/" - - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: "true" - readinessProbe: - failureThreshold: 60 - httpGet: - #path: /robots.txt - path: /login - port: 3000 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 1 - successThreshold: 1 - timeoutSeconds: 2 - livenessProbe: - failureThreshold: 60 - initialDelaySeconds: 1 - periodSeconds: 1 - successThreshold: 1 - tcpSocket: - port: 3000 - timeoutSeconds: 1 - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 500m - memory: 1024Mi + - name: server + image: labs.etsi.org:5050/tfs/controller/webui:latest + imagePullPolicy: Always + ports: + - containerPort: 8004 + env: + - name: LOG_LEVEL + value: "INFO" + - name: WEBUISERVICE_SERVICE_BASEURL_HTTP + value: "/webui/" + readinessProbe: + httpGet: + path: /healthz/ready + port: 8004 + initialDelaySeconds: 5 + timeoutSeconds: 1 + livenessProbe: + httpGet: + path: /healthz/live + port: 8004 + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi + - name: grafana + image: grafana/grafana:8.5.22 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http-grafana + protocol: TCP + env: + - name: GF_SERVER_ROOT_URL + value: "http://0.0.0.0:3000/grafana/" + - name: GF_SERVER_SERVE_FROM_SUB_PATH + value: "true" + readinessProbe: + failureThreshold: 60 + httpGet: + #path: /robots.txt + path: /login + port: 3000 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + failureThreshold: 60 + initialDelaySeconds: 1 + periodSeconds: 1 + successThreshold: 1 + tcpSocket: + port: 3000 + timeoutSeconds: 1 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1024Mi --- apiVersion: v1 kind: Service @@ -111,9 +111,9 @@ spec: selector: app: webuiservice ports: - - name: webui - port: 8004 - targetPort: 8004 - - name: grafana - port: 3000 - targetPort: 3000 + - name: webui + port: 8004 + targetPort: 8004 + - name: grafana + port: 3000 + targetPort: 3000 diff --git a/manifests/ztpservice.yaml b/manifests/ztpservice.yaml index e2be80cea..323d3c4bc 100644 --- a/manifests/ztpservice.yaml +++ b/manifests/ztpservice.yaml @@ -117,9 +117,9 @@ spec: minReplicas: 1 maxReplicas: 10 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 -- GitLab From 57b355ebd49f8eff303386e40de20cde1a19c1d9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:25:00 +0000 Subject: [PATCH 52/73] GitLab CI/CD pipeline - ECOC'22 - Cleanup for final testing --- src/tests/ecoc22/.gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 9f20076bf..f3fb8d098 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -54,12 +54,12 @@ end2end_test ecoc22: - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="" -- GitLab From 12a1639859a5ad5914795056f63f64ffbf9698c0 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:26:36 +0000 Subject: [PATCH 53/73] Context component: - Fixed set/update condition for components - Removed unneeded log messages --- src/context/service/database/Device.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 7f58ddeb5..d0674e89b 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -203,15 +203,15 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi stmt = stmt.returning(TopologyDeviceModel.topology_uuid) topology_uuids = session.execute(stmt).fetchall() - LOGGER.warning('RAW topology_uuids={:s}'.format(str(topology_uuids))) + #LOGGER.warning('RAW topology_uuids={:s}'.format(str(topology_uuids))) if len(topology_uuids) > 0: topology_uuids = [topology_uuid[0] for topology_uuid in topology_uuids] - LOGGER.warning('NEW topology_uuids={:s}'.format(str(topology_uuids))) + #LOGGER.warning('NEW topology_uuids={:s}'.format(str(topology_uuids))) query = session.query(TopologyModel) query = query.filter(TopologyModel.topology_uuid.in_(topology_uuids)) device_topologies : List[TopologyModel] = query.all() device_topology_ids = [obj.dump_id() for obj in device_topologies] - LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids))) + #LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids))) updated_components = False @@ -233,7 +233,7 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid) - return updated or updated_endpoints or changed_config_rules, device_topology_ids + return updated or updated_endpoints or updated_components or changed_config_rules, device_topology_ids updated, device_topology_ids = run_transaction(sessionmaker(bind=db_engine), callback) device_id = json_device_id(device_uuid) -- GitLab From fb89be3b2d8b8fb57c151df68c5465f3cbd7a85c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:31:18 +0000 Subject: [PATCH 54/73] GitLab CI/CD pipeline - ECOC'22 & Service component: - Testing failure condition --- .../service/ServiceServiceServicerImpl.py | 22 +++++++++---------- src/tests/ecoc22/.gitlab-ci.yml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index b5623885f..37f72efdc 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -312,19 +312,19 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): pathcomp_request = PathCompRequest() pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member - if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : - pathcomp_request.shortest_path.Clear() # pylint: disable=no-member - else: - pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member + if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : + pathcomp_request.shortest_path.Clear() # pylint: disable=no-member + else: + pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member - pathcomp = PathCompClient() - pathcomp_reply = pathcomp.Compute(pathcomp_request) - pathcomp.close() + pathcomp = PathCompClient() + pathcomp_reply = pathcomp.Compute(pathcomp_request) + pathcomp.close() - # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among - # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be - # executed) to implement the requested create/update operation. - tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) + # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among + # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be + # executed) to implement the requested create/update operation. + tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) tasks_scheduler.execute_all() return service_with_uuids.service_id diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index f3fb8d098..beefc4722 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -54,6 +54,7 @@ end2end_test ecoc22: - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml @@ -85,7 +86,6 @@ end2end_test ecoc22: --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - - docker logs ${TEST_NAME} - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend -- GitLab From 86279da98a5216ce1d119bd751b6268b464f729a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:35:00 +0000 Subject: [PATCH 55/73] Service component: - Recovering manually-forced error used for testing end-to-end integration tests --- .../service/ServiceServiceServicerImpl.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 37f72efdc..b5623885f 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -312,19 +312,19 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): pathcomp_request = PathCompRequest() pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member - if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : - pathcomp_request.shortest_path.Clear() # pylint: disable=no-member - else: - pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member + if num_disjoint_paths is None or num_disjoint_paths in {0, 1} : + pathcomp_request.shortest_path.Clear() # pylint: disable=no-member + else: + pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member - pathcomp = PathCompClient() - pathcomp_reply = pathcomp.Compute(pathcomp_request) - pathcomp.close() + pathcomp = PathCompClient() + pathcomp_reply = pathcomp.Compute(pathcomp_request) + pathcomp.close() - # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among - # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be - # executed) to implement the requested create/update operation. - tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) + # Feed TaskScheduler with this path computation reply. TaskScheduler identifies inter-dependencies among + # the services and connections retrieved and produces a schedule of tasks (an ordered list of tasks to be + # executed) to implement the requested create/update operation. + tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) tasks_scheduler.execute_all() return service_with_uuids.service_id -- GitLab From 892a1e9380beec788adfb607122934659021f279 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 12:37:58 +0000 Subject: [PATCH 56/73] GitLab CI/CD pipeline - OFC'22: - Reactivated end-to-end test - Updated CI/CD pipeline descriptor --- src/tests/.gitlab-ci.yml | 2 +- src/tests/ofc22/.gitlab-ci.yml | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index 6a4441d19..41b8bb36c 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -14,7 +14,7 @@ # include the individual .gitlab-ci.yml of each end-to-end integration test include: - #- local: '/src/tests/ofc22/.gitlab-ci.yml' + - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 9598d4eba..ee62de632 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -54,6 +54,14 @@ end2end_test ofc22: - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="" @@ -74,16 +82,11 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - - docker create --name ${TEST_NAME} --network=host -v "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - - docker cp ./tfs_runtime_env_vars.sh ${TEST_NAME}:/var/teraflow/ - - docker start ${TEST_NAME} - - docker wait ${TEST_NAME} - - docker logs ${TEST_NAME} - #- source tfs_runtime_env_vars.sh - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_bootstrap.py --junitxml=./src/tests/${TEST_NAME}/report_bootstrap.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_create_service.py --junitxml=./src/tests/${TEST_NAME}/report_create_service.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_delete_service.py --junitxml=./src/tests/${TEST_NAME}/report_delete_service.xml - #- pytest --verbose --log-level=INFO ./src/tests/${TEST_NAME}/tests/test_functional_cleanup.py --junitxml=./src/tests/${TEST_NAME}/report_cleanup.xml + - > + docker run -t --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend -- GitLab From 63b4503a302e7223ab4d8d56caaa7b9d8fe34f33 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 18:35:02 +0000 Subject: [PATCH 57/73] Framework - Mutex Queues: - Added logs to identify issue --- src/common/tools/mutex_queues/MutexQueues.py | 39 +++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/common/tools/mutex_queues/MutexQueues.py b/src/common/tools/mutex_queues/MutexQueues.py index 96e22a86f..b80dd1a9c 100644 --- a/src/common/tools/mutex_queues/MutexQueues.py +++ b/src/common/tools/mutex_queues/MutexQueues.py @@ -34,10 +34,12 @@ # driver.configure(settings) # self.mutex_queues.signal_done(device_uuid) -import threading +import logging, threading from queue import Queue, Empty from typing import Dict +LOGGER = logging.getLogger(__name__) + class MutexQueues: def __init__(self) -> None: # lock to protect dictionary updates @@ -46,36 +48,63 @@ class MutexQueues: # dictionaty of queues of mutexes: queue_name => queue[mutex] # first mutex is the running one self.mutex_queues : Dict[str, Queue[threading.Event]] = dict() - + + def add_alias(self, queue_name_a : str, queue_name_b : str) -> None: + with self.lock: + if queue_name_a in self.mutex_queues and queue_name_b not in self.mutex_queues: + self.mutex_queues[queue_name_b] = self.mutex_queues[queue_name_a] + elif queue_name_b in self.mutex_queues and queue_name_a not in self.mutex_queues: + self.mutex_queues[queue_name_a] = self.mutex_queues[queue_name_b] + def wait_my_turn(self, queue_name : str) -> None: + LOGGER.warning('[wait_my_turn] begin queue_name={:s}'.format(str(queue_name))) # create my mutex and enqueue it mutex = threading.Event() + LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex={:s}'.format(str(queue_name), str(mutex))) with self.lock: + LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex_queues={:s}'.format(str(queue_name), str(self.mutex_queues))) queue : Queue = self.mutex_queues.setdefault(queue_name, Queue()) first_in_queue = (queue.qsize() == 0) + LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} first_in_queue={:s}'.format(str(queue_name), str(first_in_queue))) queue.put_nowait(mutex) # if I'm the first in the queue upon addition, means there are no running tasks # directly return without waiting - if first_in_queue: return + if first_in_queue: + LOGGER.warning('[wait_my_turn] end first_in_queue queue_name={:s}'.format(str(queue_name))) + return # otherwise, wait for my turn in the queue + LOGGER.warning('[wait_my_turn] waiting queue_name={:s}'.format(str(queue_name))) mutex.wait() + LOGGER.warning('[wait_my_turn] end wait queue_name={:s}'.format(str(queue_name))) def signal_done(self, queue_name : str) -> None: + LOGGER.warning('[signal_done] begin queue_name={:s}'.format(str(queue_name))) # I'm done with my work with self.lock: + LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex_queues={:s}'.format(str(queue_name), str(self.mutex_queues))) queue : Queue = self.mutex_queues.setdefault(queue_name, Queue()) + LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} queue={:s}'.format(str(queue_name), str(queue))) # remove myself from the queue try: - queue.get(block=True, timeout=0.1) + LOGGER.warning('[wait_my_turn] [lock] before get queue_name={:s}'.format(str(queue_name))) + mutex = queue.get(block=True, timeout=0.1) + LOGGER.warning('[wait_my_turn] [lock] after get queue_name={:s} mutex={:s}'.format(str(queue_name), str(mutex))) except Empty: + LOGGER.warning('[wait_my_turn] [lock] empty queue_name={:s}'.format(str(queue_name))) pass # if there are no other tasks queued, return - if queue.qsize() == 0: return + if queue.qsize() == 0: + LOGGER.warning('[wait_my_turn] end queue.qsize==0 queue_name={:s}'.format(str(queue_name))) + return # otherwise, signal the next task in the queue to start next_mutex : threading.Event = queue.queue[0] + LOGGER.warning('[wait_my_turn] [lock] before set queue_name={:s} next_mutex={:s}'.format(str(queue_name), str(next_mutex))) next_mutex.set() + LOGGER.warning('[wait_my_turn] [lock] after set queue_name={:s} next_mutex={:s}'.format(str(queue_name), str(next_mutex))) + + LOGGER.warning('[signal_done] end set queue_name={:s}'.format(str(queue_name))) -- GitLab From 9d2f3119418a516d9038fa79817760d3760548a8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 18:35:15 +0000 Subject: [PATCH 58/73] GitLab CI/CD pipeline - OFC'22: - Deactivated deploy of WebUI --- src/tests/ofc22/deploy_specs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/deploy_specs.sh b/src/tests/ofc22/deploy_specs.sh index aad41f33d..ee75575ab 100755 --- a/src/tests/ofc22/deploy_specs.sh +++ b/src/tests/ofc22/deploy_specs.sh @@ -21,7 +21,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. #export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" -export TFS_COMPONENTS="context device pathcomp service slice nbi webui" +export TFS_COMPONENTS="context device pathcomp service slice nbi" # Uncomment to activate Monitoring export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" -- GitLab From 786a54f3595d8ab7c7434b3bb92da77875ce9c5d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 20 Mar 2024 18:35:39 +0000 Subject: [PATCH 59/73] Device component: - Added logs to identify issue --- .../service/DeviceServiceServicerImpl.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index 5e5d7540c..da152b1d3 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -88,10 +88,17 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): # update device_uuid to honor UUID provided by Context device_uuid = device.device_id.device_uuid.uuid + device_name = device.name t2 = time.time() + LOGGER.warning('[AddDevice] before add_alias {:s} {:s}'.format(str(device_uuid), str(device_name))) + self.mutex_queues.add_alias(device_uuid, device_name) + LOGGER.warning('[AddDevice] after add_alias {:s} {:s}'.format(str(device_uuid), str(device_name))) + + LOGGER.warning('[AddDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) + LOGGER.warning('[AddDevice] after wait_my_turn {:s}'.format(str(device_uuid))) t3 = time.time() try: driver : _Driver = get_driver(self.driver_instance_cache, device) @@ -202,7 +209,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_id finally: + LOGGER.warning('[AddDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) + LOGGER.warning('[AddDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ConfigureDevice(self, request : Device, context : grpc.ServicerContext) -> DeviceId: @@ -210,7 +219,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): device_id = request.device_id device_uuid = device_id.device_uuid.uuid + LOGGER.warning('[ConfigureDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) + LOGGER.warning('[ConfigureDevice] after wait_my_turn {:s}'.format(str(device_uuid))) t1 = time.time() try: context_client = ContextClient() @@ -294,13 +305,17 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_id finally: + LOGGER.warning('[ConfigureDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) + LOGGER.warning('[ConfigureDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteDevice(self, request : DeviceId, context : grpc.ServicerContext) -> Empty: device_uuid = request.device_uuid.uuid + LOGGER.warning('[DeleteDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) + LOGGER.warning('[DeleteDevice] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -315,13 +330,17 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): context_client.RemoveDevice(request) return Empty() finally: + LOGGER.warning('[DeleteDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) + LOGGER.warning('[DeleteDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetInitialConfig(self, request : DeviceId, context : grpc.ServicerContext) -> DeviceConfig: device_uuid = request.device_uuid.uuid + LOGGER.warning('[GetInitialConfig] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) + LOGGER.warning('[GetInitialConfig] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -344,7 +363,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_config finally: + LOGGER.warning('[GetInitialConfig] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) + LOGGER.warning('[GetInitialConfig] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def MonitorDeviceKpi(self, request : MonitoringSettings, context : grpc.ServicerContext) -> Empty: @@ -362,7 +383,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): raise OperationFailedException('MonitorDeviceKpi', extra_details=msg) device_uuid = kpi_details[0] + LOGGER.warning('[MonitorDeviceKpi] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) + LOGGER.warning('[MonitorDeviceKpi] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -381,4 +404,6 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return Empty() finally: + LOGGER.warning('[MonitorDeviceKpi] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) + LOGGER.warning('[MonitorDeviceKpi] after signal_done {:s}'.format(str(device_uuid))) -- GitLab From be41ac9006b877ac2b6a6bf5f0aa54151aeccd4a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 21 Mar 2024 08:32:25 +0000 Subject: [PATCH 60/73] GitLab CI/CD pipeline: - Deactivated ECOC'22 for troubleshooting --- src/tests/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index 41b8bb36c..d48456b83 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - - local: '/src/tests/ecoc22/.gitlab-ci.yml' + #- local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' #- local: '/src/tests/ofc23/.gitlab-ci.yml' #- local: '/src/tests/ofc24/.gitlab-ci.yml' -- GitLab From 1bd035a0f63dcc4da4641aa736dbf0a0d2f9e0fe Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Thu, 21 Mar 2024 22:38:26 +0000 Subject: [PATCH 61/73] CI/CD pipeline - OFC'22 added code to kill test after 2 minutes --- src/tests/ofc22/Dockerfile | 2 +- src/tests/ofc22/tests/conftest.py | 63 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/tests/ofc22/tests/conftest.py diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 4817bd93a..93b4a77d0 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -93,7 +93,7 @@ export PYTHONPATH=/var/teraflow pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_bootstrap.py --junitxml=/opt/results/report_bootstrap.xml pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_create_service.py --junitxml=/opt/results/report_create_service.xml pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_delete_service.py --junitxml=/opt/results/report_delete_service.xml -pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml +pytest --verbose --timeout=120 --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml EOF RUN chmod ug+x ./run_tests.sh diff --git a/src/tests/ofc22/tests/conftest.py b/src/tests/ofc22/tests/conftest.py new file mode 100644 index 000000000..6650864b0 --- /dev/null +++ b/src/tests/ofc22/tests/conftest.py @@ -0,0 +1,63 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Acknowledgement: https://stackoverflow.com/questions/46766899/pytest-timeout-fail-test-instead-killing-whole-test-run + +import pytest, signal + + +class Termination(SystemExit): + pass + + +class TimeoutExit(BaseException): + pass + + +def _terminate(signum, frame): + raise Termination("Runner is terminated from outside.") + + +def _timeout(signum, frame): + raise TimeoutExit("Runner timeout is reached, runner is terminating.") + + +@pytest.hookimpl +def pytest_addoption(parser): + parser.addoption( + '--timeout', action='store', dest='timeout', type=int, default=None, + help="number of seconds before each test failure") + + +@pytest.hookimpl +def pytest_configure(config): + # Install the signal handlers that we want to process. + signal.signal(signal.SIGTERM, _terminate) + signal.signal(signal.SIGALRM, _timeout) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + + # Set the per-test timeout (an alarm signal). + if item.config.option.timeout is not None: + signal.alarm(item.config.option.timeout) + + try: + # Run the setup, test body, and teardown stages. + yield + finally: + # Disable the alarm when the test passes or fails. + # I.e. when we get into the framework's body. + signal.alarm(0) -- GitLab From f44bef6233d760e688d22993656fe0b0284f0afa Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Thu, 21 Mar 2024 22:48:39 +0000 Subject: [PATCH 62/73] CI/CD pipeline - OFC'22: - Moved logs to after_script --- src/tests/ofc22/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index ee62de632..1806a065c 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -87,6 +87,7 @@ end2end_test ofc22: --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend @@ -95,7 +96,6 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server - after_script: - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' -- GitLab From e942151fcd19f64b675868dd6b0c9bfd31e648a6 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Thu, 21 Mar 2024 23:10:23 +0000 Subject: [PATCH 63/73] CI/CD pipeline - OFC'22 --- src/tests/ofc22/.gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 1806a065c..1a756a650 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -88,14 +88,14 @@ end2end_test ofc22: --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server + - kubectl --namespace tfs logs deployment/contextservice -c server + - kubectl --namespace tfs logs deployment/deviceservice -c server + - kubectl --namespace tfs logs deployment/pathcompservice -c frontend + - kubectl --namespace tfs logs deployment/serviceservice -c server + - kubectl --namespace tfs logs deployment/sliceservice -c server + - kubectl --namespace tfs logs deployment/nbiservice -c server + - kubectl --namespace tfs logs deployment/monitoringservice -c server + - kubectl --namespace tfs logs deployment/ztpservice -c server - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' -- GitLab From 3d5f1aa3bcd7e5fe106c56496195e76785e8fabe Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Thu, 21 Mar 2024 23:54:23 +0000 Subject: [PATCH 64/73] CI/CD pipeline - OFC'22 --- src/common/tools/mutex_queues/MutexQueues.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/tools/mutex_queues/MutexQueues.py b/src/common/tools/mutex_queues/MutexQueues.py index b80dd1a9c..c29194ca8 100644 --- a/src/common/tools/mutex_queues/MutexQueues.py +++ b/src/common/tools/mutex_queues/MutexQueues.py @@ -55,6 +55,8 @@ class MutexQueues: self.mutex_queues[queue_name_b] = self.mutex_queues[queue_name_a] elif queue_name_b in self.mutex_queues and queue_name_a not in self.mutex_queues: self.mutex_queues[queue_name_a] = self.mutex_queues[queue_name_b] + elif queue_name_b not in self.mutex_queues and queue_name_a not in self.mutex_queues: + self.mutex_queues[queue_name_b] = self.mutex_queues.setdefault(queue_name_a, Queue()) def wait_my_turn(self, queue_name : str) -> None: LOGGER.warning('[wait_my_turn] begin queue_name={:s}'.format(str(queue_name))) -- GitLab From 652165312ea4ab8f4427dff94ddb0139b2207f88 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Fri, 22 Mar 2024 00:21:00 +0000 Subject: [PATCH 65/73] Pre-merge cleanup --- .gitlab-ci.yml | 32 +++++----- src/common/tools/mutex_queues/MutexQueues.py | 28 +-------- .../service/DeviceServiceServicerImpl.py | 23 ------- src/tests/.gitlab-ci.yml | 2 +- src/tests/ecoc22/.gitlab-ci.yml | 11 ++-- src/tests/ecoc22/tests/conftest.py | 63 +++++++++++++++++++ src/tests/ofc22/.gitlab-ci.yml | 39 ++++++------ src/tests/ofc22/Dockerfile | 2 +- src/tests/ofc22/deploy_specs.sh | 2 +- 9 files changed, 111 insertions(+), 91 deletions(-) create mode 100644 src/tests/ecoc22/tests/conftest.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a454253f..064cedd15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,28 +22,28 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - ###- local: '/src/monitoring/.gitlab-ci.yml' - ###- local: '/src/nbi/.gitlab-ci.yml' - ###- local: '/src/context/.gitlab-ci.yml' - ###- local: '/src/device/.gitlab-ci.yml' - ###- local: '/src/service/.gitlab-ci.yml' - ###- local: '/src/dbscanserving/.gitlab-ci.yml' - ###- local: '/src/opticalattackmitigator/.gitlab-ci.yml' - ###- local: '/src/opticalattackdetector/.gitlab-ci.yml' - ###- local: '/src/opticalattackmanager/.gitlab-ci.yml' - ###- local: '/src/ztp/.gitlab-ci.yml' - ###- local: '/src/policy/.gitlab-ci.yml' - ###- local: '/src/forecaster/.gitlab-ci.yml' + - local: '/src/monitoring/.gitlab-ci.yml' + - local: '/src/nbi/.gitlab-ci.yml' + - local: '/src/context/.gitlab-ci.yml' + - local: '/src/device/.gitlab-ci.yml' + - local: '/src/service/.gitlab-ci.yml' + - local: '/src/dbscanserving/.gitlab-ci.yml' + - local: '/src/opticalattackmitigator/.gitlab-ci.yml' + - local: '/src/opticalattackdetector/.gitlab-ci.yml' + - local: '/src/opticalattackmanager/.gitlab-ci.yml' + - local: '/src/ztp/.gitlab-ci.yml' + - local: '/src/policy/.gitlab-ci.yml' + - local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - ###- local: '/src/slice/.gitlab-ci.yml' + - local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - ###- local: '/src/pathcomp/.gitlab-ci.yml' + - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - ###- local: '/src/load_generator/.gitlab-ci.yml' - ###- local: '/src/bgpls_speaker/.gitlab-ci.yml' + - local: '/src/load_generator/.gitlab-ci.yml' + - local: '/src/bgpls_speaker/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/common/tools/mutex_queues/MutexQueues.py b/src/common/tools/mutex_queues/MutexQueues.py index c29194ca8..50074cd4a 100644 --- a/src/common/tools/mutex_queues/MutexQueues.py +++ b/src/common/tools/mutex_queues/MutexQueues.py @@ -34,12 +34,10 @@ # driver.configure(settings) # self.mutex_queues.signal_done(device_uuid) -import logging, threading +import threading from queue import Queue, Empty from typing import Dict -LOGGER = logging.getLogger(__name__) - class MutexQueues: def __init__(self) -> None: # lock to protect dictionary updates @@ -59,54 +57,34 @@ class MutexQueues: self.mutex_queues[queue_name_b] = self.mutex_queues.setdefault(queue_name_a, Queue()) def wait_my_turn(self, queue_name : str) -> None: - LOGGER.warning('[wait_my_turn] begin queue_name={:s}'.format(str(queue_name))) # create my mutex and enqueue it mutex = threading.Event() - LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex={:s}'.format(str(queue_name), str(mutex))) with self.lock: - LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex_queues={:s}'.format(str(queue_name), str(self.mutex_queues))) queue : Queue = self.mutex_queues.setdefault(queue_name, Queue()) first_in_queue = (queue.qsize() == 0) - LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} first_in_queue={:s}'.format(str(queue_name), str(first_in_queue))) queue.put_nowait(mutex) # if I'm the first in the queue upon addition, means there are no running tasks # directly return without waiting - if first_in_queue: - LOGGER.warning('[wait_my_turn] end first_in_queue queue_name={:s}'.format(str(queue_name))) - return + if first_in_queue: return # otherwise, wait for my turn in the queue - LOGGER.warning('[wait_my_turn] waiting queue_name={:s}'.format(str(queue_name))) mutex.wait() - LOGGER.warning('[wait_my_turn] end wait queue_name={:s}'.format(str(queue_name))) def signal_done(self, queue_name : str) -> None: - LOGGER.warning('[signal_done] begin queue_name={:s}'.format(str(queue_name))) # I'm done with my work with self.lock: - LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} mutex_queues={:s}'.format(str(queue_name), str(self.mutex_queues))) queue : Queue = self.mutex_queues.setdefault(queue_name, Queue()) - LOGGER.warning('[wait_my_turn] [lock] queue_name={:s} queue={:s}'.format(str(queue_name), str(queue))) # remove myself from the queue try: - LOGGER.warning('[wait_my_turn] [lock] before get queue_name={:s}'.format(str(queue_name))) mutex = queue.get(block=True, timeout=0.1) - LOGGER.warning('[wait_my_turn] [lock] after get queue_name={:s} mutex={:s}'.format(str(queue_name), str(mutex))) except Empty: - LOGGER.warning('[wait_my_turn] [lock] empty queue_name={:s}'.format(str(queue_name))) pass # if there are no other tasks queued, return - if queue.qsize() == 0: - LOGGER.warning('[wait_my_turn] end queue.qsize==0 queue_name={:s}'.format(str(queue_name))) - return + if queue.qsize() == 0: return # otherwise, signal the next task in the queue to start next_mutex : threading.Event = queue.queue[0] - LOGGER.warning('[wait_my_turn] [lock] before set queue_name={:s} next_mutex={:s}'.format(str(queue_name), str(next_mutex))) next_mutex.set() - LOGGER.warning('[wait_my_turn] [lock] after set queue_name={:s} next_mutex={:s}'.format(str(queue_name), str(next_mutex))) - - LOGGER.warning('[signal_done] end set queue_name={:s}'.format(str(queue_name))) diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index da152b1d3..4f0ba5722 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -92,13 +92,8 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): t2 = time.time() - LOGGER.warning('[AddDevice] before add_alias {:s} {:s}'.format(str(device_uuid), str(device_name))) self.mutex_queues.add_alias(device_uuid, device_name) - LOGGER.warning('[AddDevice] after add_alias {:s} {:s}'.format(str(device_uuid), str(device_name))) - - LOGGER.warning('[AddDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) - LOGGER.warning('[AddDevice] after wait_my_turn {:s}'.format(str(device_uuid))) t3 = time.time() try: driver : _Driver = get_driver(self.driver_instance_cache, device) @@ -209,9 +204,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_id finally: - LOGGER.warning('[AddDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) - LOGGER.warning('[AddDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ConfigureDevice(self, request : Device, context : grpc.ServicerContext) -> DeviceId: @@ -219,9 +212,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): device_id = request.device_id device_uuid = device_id.device_uuid.uuid - LOGGER.warning('[ConfigureDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) - LOGGER.warning('[ConfigureDevice] after wait_my_turn {:s}'.format(str(device_uuid))) t1 = time.time() try: context_client = ContextClient() @@ -305,17 +296,13 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_id finally: - LOGGER.warning('[ConfigureDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) - LOGGER.warning('[ConfigureDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteDevice(self, request : DeviceId, context : grpc.ServicerContext) -> Empty: device_uuid = request.device_uuid.uuid - LOGGER.warning('[DeleteDevice] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) - LOGGER.warning('[DeleteDevice] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -330,17 +317,13 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): context_client.RemoveDevice(request) return Empty() finally: - LOGGER.warning('[DeleteDevice] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) - LOGGER.warning('[DeleteDevice] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetInitialConfig(self, request : DeviceId, context : grpc.ServicerContext) -> DeviceConfig: device_uuid = request.device_uuid.uuid - LOGGER.warning('[GetInitialConfig] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) - LOGGER.warning('[GetInitialConfig] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -363,9 +346,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return device_config finally: - LOGGER.warning('[GetInitialConfig] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) - LOGGER.warning('[GetInitialConfig] after signal_done {:s}'.format(str(device_uuid))) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def MonitorDeviceKpi(self, request : MonitoringSettings, context : grpc.ServicerContext) -> Empty: @@ -383,9 +364,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): raise OperationFailedException('MonitorDeviceKpi', extra_details=msg) device_uuid = kpi_details[0] - LOGGER.warning('[MonitorDeviceKpi] before wait_my_turn {:s}'.format(str(device_uuid))) self.mutex_queues.wait_my_turn(device_uuid) - LOGGER.warning('[MonitorDeviceKpi] after wait_my_turn {:s}'.format(str(device_uuid))) try: context_client = ContextClient() device = get_device( @@ -404,6 +383,4 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): return Empty() finally: - LOGGER.warning('[MonitorDeviceKpi] before signal_done {:s}'.format(str(device_uuid))) self.mutex_queues.signal_done(device_uuid) - LOGGER.warning('[MonitorDeviceKpi] after signal_done {:s}'.format(str(device_uuid))) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index d48456b83..41b8bb36c 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - local: '/src/tests/ofc22/.gitlab-ci.yml' #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - #- local: '/src/tests/ecoc22/.gitlab-ci.yml' + - local: '/src/tests/ecoc22/.gitlab-ci.yml' #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' #- local: '/src/tests/ofc23/.gitlab-ci.yml' #- local: '/src/tests/ofc24/.gitlab-ci.yml' diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index beefc4722..d677cb823 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -62,10 +62,10 @@ end2end_test ecoc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh - #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - #- export TFS_SKIP_BUILD="" - #- export TFS_IMAGE_TAG="latest" - #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + - export TFS_SKIP_BUILD="YES" + - export TFS_IMAGE_TAG="latest" + - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh @@ -86,13 +86,14 @@ end2end_test ecoc22: --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: + - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - after_script: - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' diff --git a/src/tests/ecoc22/tests/conftest.py b/src/tests/ecoc22/tests/conftest.py new file mode 100644 index 000000000..6650864b0 --- /dev/null +++ b/src/tests/ecoc22/tests/conftest.py @@ -0,0 +1,63 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Acknowledgement: https://stackoverflow.com/questions/46766899/pytest-timeout-fail-test-instead-killing-whole-test-run + +import pytest, signal + + +class Termination(SystemExit): + pass + + +class TimeoutExit(BaseException): + pass + + +def _terminate(signum, frame): + raise Termination("Runner is terminated from outside.") + + +def _timeout(signum, frame): + raise TimeoutExit("Runner timeout is reached, runner is terminating.") + + +@pytest.hookimpl +def pytest_addoption(parser): + parser.addoption( + '--timeout', action='store', dest='timeout', type=int, default=None, + help="number of seconds before each test failure") + + +@pytest.hookimpl +def pytest_configure(config): + # Install the signal handlers that we want to process. + signal.signal(signal.SIGTERM, _terminate) + signal.signal(signal.SIGALRM, _timeout) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + + # Set the per-test timeout (an alarm signal). + if item.config.option.timeout is not None: + signal.alarm(item.config.option.timeout) + + try: + # Run the setup, test body, and teardown stages. + yield + finally: + # Disable the alarm when the test passes or fails. + # I.e. when we get into the framework's body. + signal.alarm(0) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 1a756a650..e9c8c0c03 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -55,18 +55,18 @@ end2end_test ofc22: # Configure TeraFlowSDN deployment # Uncomment if DEBUG log level is needed for the components - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - - yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh - #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - #- export TFS_SKIP_BUILD="" - #- export TFS_IMAGE_TAG="latest" - #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + - export TFS_SKIP_BUILD="YES" + - export TFS_IMAGE_TAG="latest" + - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh @@ -88,14 +88,15 @@ end2end_test ofc22: --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: - - kubectl --namespace tfs logs deployment/contextservice -c server - - kubectl --namespace tfs logs deployment/deviceservice -c server - - kubectl --namespace tfs logs deployment/pathcompservice -c frontend - - kubectl --namespace tfs logs deployment/serviceservice -c server - - kubectl --namespace tfs logs deployment/sliceservice -c server - - kubectl --namespace tfs logs deployment/nbiservice -c server - - kubectl --namespace tfs logs deployment/monitoringservice -c server - - kubectl --namespace tfs logs deployment/ztpservice -c server + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c server - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - docker images --filter="dangling=true" --quiet | xargs -r docker rmi #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' diff --git a/src/tests/ofc22/Dockerfile b/src/tests/ofc22/Dockerfile index 93b4a77d0..4817bd93a 100644 --- a/src/tests/ofc22/Dockerfile +++ b/src/tests/ofc22/Dockerfile @@ -93,7 +93,7 @@ export PYTHONPATH=/var/teraflow pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_bootstrap.py --junitxml=/opt/results/report_bootstrap.xml pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_create_service.py --junitxml=/opt/results/report_create_service.xml pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_delete_service.py --junitxml=/opt/results/report_delete_service.xml -pytest --verbose --timeout=120 --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml +pytest --verbose --log-level=INFO /var/teraflow/tests/ofc22/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml EOF RUN chmod ug+x ./run_tests.sh diff --git a/src/tests/ofc22/deploy_specs.sh b/src/tests/ofc22/deploy_specs.sh index ee75575ab..aad41f33d 100755 --- a/src/tests/ofc22/deploy_specs.sh +++ b/src/tests/ofc22/deploy_specs.sh @@ -21,7 +21,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. #export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" -export TFS_COMPONENTS="context device pathcomp service slice nbi" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" -- GitLab From 5128fd5f6deb2d3f4b2c3382d59368eee8912df1 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Fri, 22 Mar 2024 00:23:50 +0000 Subject: [PATCH 66/73] Pre-merge cleanup --- src/common/tools/mutex_queues/MutexQueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tools/mutex_queues/MutexQueues.py b/src/common/tools/mutex_queues/MutexQueues.py index 50074cd4a..e18432500 100644 --- a/src/common/tools/mutex_queues/MutexQueues.py +++ b/src/common/tools/mutex_queues/MutexQueues.py @@ -78,7 +78,7 @@ class MutexQueues: # remove myself from the queue try: - mutex = queue.get(block=True, timeout=0.1) + queue.get(block=True, timeout=0.1) except Empty: pass -- GitLab From a3b6992cc50342ed15e82842065558c4fafda452 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 09:34:35 +0000 Subject: [PATCH 67/73] GitLab CI/CD pipeline & reorganization: - Updated build commands to docker buildx - Removed unneeded scripts - Moved scripts to scripts folder --- deploy/component.sh | 25 +++++++++++--- deploy/mock_blockchain.sh | 17 +++++++++- .../expose_ingress_grpc.sh | 0 .../run_tests_docker.sh | 19 +++++++++-- {src => scripts}/start_webui_dev_mode.sh | 0 scripts/wait_context_nats.sh | 28 ++++++++++++++++ src/bgpls_speaker/.gitlab-ci.yml | 2 +- src/build.sh | 33 ------------------- src/clean.sh | 19 ----------- src/context/.gitlab-ci.yml | 2 +- src/dbscanserving/.gitlab-ci.yml | 2 +- src/device/.gitlab-ci.yml | 2 +- src/dlt/.gitlab-ci.yml | 4 +-- src/e2e_orchestrator/.gitlab-ci.yml | 2 +- src/forecaster/.gitlab-ci.yml | 2 +- src/interdomain/.gitlab-ci.yml | 2 +- src/l3_attackmitigator/.gitlab-ci.yml | 2 +- .../.gitlab-ci.yml | 2 +- .../.gitlab-ci.yml | 2 +- src/load_generator/.gitlab-ci.yml | 2 +- src/monitoring/.gitlab-ci.yml | 2 +- src/nbi/.gitlab-ci.yml | 2 +- src/opticalattackdetector/.gitlab-ci.yml | 2 +- src/opticalattackmanager/.gitlab-ci.yml | 2 +- src/opticalattackmitigator/.gitlab-ci.yml | 2 +- src/pathcomp/.gitlab-ci.yml | 6 ++-- src/policy/.gitlab-ci.yml | 6 ++-- src/service/.gitlab-ci.yml | 2 +- src/slice/.gitlab-ci.yml | 2 +- src/start.sh | 18 ---------- src/tests/ecoc22/.gitlab-ci.yml | 2 +- src/tests/ofc22/.gitlab-ci.yml | 2 +- src/webui/.gitlab-ci.yml | 2 +- src/ztp/.gitlab-ci.yml | 6 ++-- 34 files changed, 113 insertions(+), 110 deletions(-) rename expose_ingress_grpc.sh => scripts/expose_ingress_grpc.sh (100%) rename run_tests_docker.sh => scripts/run_tests_docker.sh (72%) rename {src => scripts}/start_webui_dev_mode.sh (100%) create mode 100755 scripts/wait_context_nats.sh delete mode 100755 src/build.sh delete mode 100755 src/clean.sh delete mode 100755 src/start.sh diff --git a/deploy/component.sh b/deploy/component.sh index e26488186..d3a94c259 100755 --- a/deploy/component.sh +++ b/deploy/component.sh @@ -40,6 +40,21 @@ export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"} # Automated steps start here ######################################################################################################################## +DOCKER_BUILD="docker build" +DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "Docker version [0-9]+\." | grep -o -E "[0-9]+" | cut -c 1-3) +if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then + # If Docker version >= 23, build command was migrated to docker-buildx + # In Ubuntu, in practice, means to install package docker-buildx together with docker.io + # Check if docker-buildx plugin is installed + docker buildx version 1>/dev/null 2>/dev/null + if [[ $? -ne 0 ]]; then + echo "Docker buildx command is not installed. Check: https://docs.docker.com/build/architecture/#install-buildx" + echo "If you installed docker through APT package docker.io, consider installing also package docker-buildx" + exit 1; + fi + DOCKER_BUILD="docker buildx build" +fi + # Constants GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller" TMP_FOLDER="./tmp" @@ -60,17 +75,17 @@ for COMPONENT in $TFS_COMPONENTS; do BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" + $DOCKER_BUILD -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" elif [ "$COMPONENT" == "pathcomp" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" - docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" - docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" # next command is redundant, but helpful to keep cache updated between rebuilds - docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" else - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" fi if [ -n "$TFS_REGISTRY_IMAGES" ]; then diff --git a/deploy/mock_blockchain.sh b/deploy/mock_blockchain.sh index 74d62cd52..df8496bea 100755 --- a/deploy/mock_blockchain.sh +++ b/deploy/mock_blockchain.sh @@ -58,9 +58,24 @@ echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$IMAGE_TAG" IMAGE_URL=$(echo "$REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') +DOCKER_BUILD="docker build" +DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "Docker version [0-9]+\." | grep -o -E "[0-9]+" | cut -c 1-3) +if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then + # If Docker version >= 23, build command was migrated to docker-buildx + # In Ubuntu, in practice, means to install package docker-buildx together with docker.io + # Check if docker-buildx plugin is installed + docker buildx version 1>/dev/null 2>/dev/null + if [[ $? -ne 0 ]]; then + echo "Docker buildx command is not installed. Check: https://docs.docker.com/build/architecture/#install-buildx" + echo "If you installed docker through APT package docker.io, consider installing also package docker-buildx" + exit 1; + fi + DOCKER_BUILD="docker buildx build" +fi + echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" -docker build -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG" +$DOCKER_BUILD -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG" if [ -n "$REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$REGISTRY_IMAGE'..." diff --git a/expose_ingress_grpc.sh b/scripts/expose_ingress_grpc.sh similarity index 100% rename from expose_ingress_grpc.sh rename to scripts/expose_ingress_grpc.sh diff --git a/run_tests_docker.sh b/scripts/run_tests_docker.sh similarity index 72% rename from run_tests_docker.sh rename to scripts/run_tests_docker.sh index b7d897d76..23d920487 100755 --- a/run_tests_docker.sh +++ b/scripts/run_tests_docker.sh @@ -35,6 +35,21 @@ TMP_FOLDER="./tmp" TMP_LOGS_FOLDER="$TMP_FOLDER/logs" mkdir -p $TMP_LOGS_FOLDER +DOCKER_BUILD="docker build" +DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "Docker version [0-9]+\." | grep -o -E "[0-9]+" | cut -c 1-3) +if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then + # If Docker version >= 23, build command was migrated to docker-buildx + # In Ubuntu, in practice, means to install package docker-buildx together with docker.io + # Check if docker-buildx plugin is installed + docker buildx version 1>/dev/null 2>/dev/null + if [[ $? -ne 0 ]]; then + echo "Docker buildx command is not installed. Check: https://docs.docker.com/build/architecture/#install-buildx" + echo "If you installed docker through APT package docker.io, consider installing also package docker-buildx" + exit 1; + fi + DOCKER_BUILD="docker buildx build" +fi + for COMPONENT in $COMPONENTS; do echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$IMAGE_TAG" @@ -44,9 +59,9 @@ for COMPONENT in $COMPONENTS; do BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" + $DOCKER_BUILD -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" else - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/ > "$BUILD_LOG" + $DOCKER_BUILD -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/ > "$BUILD_LOG" fi if [ -n "$REGISTRY_IMAGE" ]; then diff --git a/src/start_webui_dev_mode.sh b/scripts/start_webui_dev_mode.sh similarity index 100% rename from src/start_webui_dev_mode.sh rename to scripts/start_webui_dev_mode.sh diff --git a/scripts/wait_context_nats.sh b/scripts/wait_context_nats.sh new file mode 100755 index 000000000..900595741 --- /dev/null +++ b/scripts/wait_context_nats.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done diff --git a/src/bgpls_speaker/.gitlab-ci.yml b/src/bgpls_speaker/.gitlab-ci.yml index 502025626..50b5ebf69 100644 --- a/src/bgpls_speaker/.gitlab-ci.yml +++ b/src/bgpls_speaker/.gitlab-ci.yml @@ -21,7 +21,7 @@ build bgpls_speaker: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/build.sh b/src/build.sh deleted file mode 100755 index b1a7d299e..000000000 --- a/src/build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Make folder containing the script the root folder for its execution -cd $(dirname $0) - -echo "BUILD context" -context/genproto.sh -docker build -t "context:develop" -f context/Dockerfile --quiet . -docker build -t "context:test" -f context/tests/Dockerfile --quiet . - -cd monitoring -./genproto.sh -cd .. - -echo "BUILD monitoring" -docker build -t "monitoring:dockerfile" -f monitoring/Dockerfile . - -echo "Prune unused images" -docker image prune --force diff --git a/src/clean.sh b/src/clean.sh deleted file mode 100755 index 21e887190..000000000 --- a/src/clean.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -docker rm --force monitoring -docker network remove teraflowbridge - diff --git a/src/context/.gitlab-ci.yml b/src/context/.gitlab-ci.yml index 5de4bc1fc..eb12743c3 100644 --- a/src/context/.gitlab-ci.yml +++ b/src/context/.gitlab-ci.yml @@ -21,7 +21,7 @@ build context: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/dbscanserving/.gitlab-ci.yml b/src/dbscanserving/.gitlab-ci.yml index d665bdaf2..2753e4c49 100644 --- a/src/dbscanserving/.gitlab-ci.yml +++ b/src/dbscanserving/.gitlab-ci.yml @@ -21,7 +21,7 @@ build dbscanserving: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index bcc2e05e5..83b18a122 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -21,7 +21,7 @@ build device: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/dlt/.gitlab-ci.yml b/src/dlt/.gitlab-ci.yml index df30a7b1d..7b14937a7 100644 --- a/src/dlt/.gitlab-ci.yml +++ b/src/dlt/.gitlab-ci.yml @@ -22,8 +22,8 @@ build dlt: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: # This first build tags the builder resulting image to prevent being removed by dangling image removal command - - docker build -t "${IMAGE_NAME}-gateway:$IMAGE_TAG" -f ./src/$IMAGE_NAME/gateway/Dockerfile . - - docker build -t "${IMAGE_NAME}-connector:$IMAGE_TAG" -f ./src/$IMAGE_NAME/connector/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-gateway:$IMAGE_TAG" -f ./src/$IMAGE_NAME/gateway/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-connector:$IMAGE_TAG" -f ./src/$IMAGE_NAME/connector/Dockerfile . - docker tag "${IMAGE_NAME}-gateway:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" - docker tag "${IMAGE_NAME}-connector:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-connector:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" diff --git a/src/e2e_orchestrator/.gitlab-ci.yml b/src/e2e_orchestrator/.gitlab-ci.yml index a14a215af..1dc93a8d4 100644 --- a/src/e2e_orchestrator/.gitlab-ci.yml +++ b/src/e2e_orchestrator/.gitlab-ci.yml @@ -21,7 +21,7 @@ build e2eorchestrator: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/forecaster/.gitlab-ci.yml b/src/forecaster/.gitlab-ci.yml index 09b2f8f4e..108febbf9 100644 --- a/src/forecaster/.gitlab-ci.yml +++ b/src/forecaster/.gitlab-ci.yml @@ -21,7 +21,7 @@ build forecaster: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/interdomain/.gitlab-ci.yml b/src/interdomain/.gitlab-ci.yml index 486c6d0a9..3cfd8c57e 100644 --- a/src/interdomain/.gitlab-ci.yml +++ b/src/interdomain/.gitlab-ci.yml @@ -21,7 +21,7 @@ build interdomain: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/l3_attackmitigator/.gitlab-ci.yml b/src/l3_attackmitigator/.gitlab-ci.yml index c43b553a4..997c43649 100644 --- a/src/l3_attackmitigator/.gitlab-ci.yml +++ b/src/l3_attackmitigator/.gitlab-ci.yml @@ -21,7 +21,7 @@ build l3_attackmitigator: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/l3_centralizedattackdetector/.gitlab-ci.yml b/src/l3_centralizedattackdetector/.gitlab-ci.yml index 057545eb1..ed0cb14d3 100644 --- a/src/l3_centralizedattackdetector/.gitlab-ci.yml +++ b/src/l3_centralizedattackdetector/.gitlab-ci.yml @@ -21,7 +21,7 @@ build l3_centralizedattackdetector: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/l3_distributedattackdetector/.gitlab-ci.yml b/src/l3_distributedattackdetector/.gitlab-ci.yml index 226cb6421..474039086 100644 --- a/src/l3_distributedattackdetector/.gitlab-ci.yml +++ b/src/l3_distributedattackdetector/.gitlab-ci.yml @@ -21,7 +21,7 @@ build l3_distributedattackdetector: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/load_generator/.gitlab-ci.yml b/src/load_generator/.gitlab-ci.yml index 5f2ee0d76..3980c19e5 100644 --- a/src/load_generator/.gitlab-ci.yml +++ b/src/load_generator/.gitlab-ci.yml @@ -21,7 +21,7 @@ build load_generator: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index 7c3a14975..6ee53bb60 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -21,7 +21,7 @@ build monitoring: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/nbi/.gitlab-ci.yml b/src/nbi/.gitlab-ci.yml index e0cac446a..2f7110202 100644 --- a/src/nbi/.gitlab-ci.yml +++ b/src/nbi/.gitlab-ci.yml @@ -21,7 +21,7 @@ build nbi: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/opticalattackdetector/.gitlab-ci.yml b/src/opticalattackdetector/.gitlab-ci.yml index 9b92c66fc..61f404c16 100644 --- a/src/opticalattackdetector/.gitlab-ci.yml +++ b/src/opticalattackdetector/.gitlab-ci.yml @@ -21,7 +21,7 @@ build opticalattackdetector: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/opticalattackmanager/.gitlab-ci.yml b/src/opticalattackmanager/.gitlab-ci.yml index 56d3c9d4d..1054a8520 100644 --- a/src/opticalattackmanager/.gitlab-ci.yml +++ b/src/opticalattackmanager/.gitlab-ci.yml @@ -21,7 +21,7 @@ build opticalattackmanager: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/opticalattackmitigator/.gitlab-ci.yml b/src/opticalattackmitigator/.gitlab-ci.yml index e684b94a7..3456348fa 100644 --- a/src/opticalattackmitigator/.gitlab-ci.yml +++ b/src/opticalattackmitigator/.gitlab-ci.yml @@ -21,7 +21,7 @@ build opticalattackmitigator: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/pathcomp/.gitlab-ci.yml b/src/pathcomp/.gitlab-ci.yml index 05113d0fe..cc5db9818 100644 --- a/src/pathcomp/.gitlab-ci.yml +++ b/src/pathcomp/.gitlab-ci.yml @@ -22,9 +22,9 @@ build pathcomp: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: # This first build tags the builder resulting image to prevent being removed by dangling image removal command - - docker build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . - - docker build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . - - docker build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . - docker tag "${IMAGE_NAME}-backend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" - docker tag "${IMAGE_NAME}-frontend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" diff --git a/src/policy/.gitlab-ci.yml b/src/policy/.gitlab-ci.yml index 8c326ff16..6ec51e86f 100644 --- a/src/policy/.gitlab-ci.yml +++ b/src/policy/.gitlab-ci.yml @@ -24,7 +24,7 @@ build policy: - export IMAGE_TAG=$(grep -m1 '' ./src/$IMAGE_NAME_POLICY/pom.xml | grep -oP '(?<=>).*(?=<)') - echo "IMAGE_TAG=${IMAGE_TAG}" >> ${BUILD_ENV_POLICY} - cat ${BUILD_ENV_POLICY} - - docker build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target builder + - docker buildx build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target builder after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi artifacts: @@ -50,7 +50,7 @@ unit_test policy: - docker rm ${REPORTS_CONTAINER} || true script: - echo "Running tests for image ${IMAGE_TAG}" - - docker build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target unit-test + - docker buildx build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target unit-test # Transfer JaCoCo and Surefire reports from within tests image - docker create --name ${REPORTS_CONTAINER} "$IMAGE_NAME_POLICY:$IMAGE_TAG" - mkdir -p ${REPORTS_PATH_POLICY} @@ -60,7 +60,7 @@ unit_test policy: - cat ${REPORTS_PATH_POLICY}/coverage.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/JaCoCo Coverage Total:/' - docker run -v "$(pwd)/src/${IMAGE_NAME_POLICY}:/${IMAGE_NAME_POLICY}" --rm registry.gitlab.com/haynes/jacoco2cobertura:1.0.7 python /opt/cover2cover.py ${IMAGE_NAME_POLICY}/reports/jacoco.xml ${IMAGE_NAME_POLICY}/src/main/java > ${REPORTS_PATH_POLICY}/cobertura.xml # Build final image - - docker build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target release + - docker buildx build -t "$IMAGE_NAME_POLICY:$IMAGE_TAG" -f ./src/$IMAGE_NAME_POLICY/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME_POLICY/ --target release - docker tag "$IMAGE_NAME_POLICY:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME_POLICY:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME_POLICY:$IMAGE_TAG" after_script: diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index c9ff94ac9..11fa4b07f 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -21,7 +21,7 @@ build service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/slice/.gitlab-ci.yml b/src/slice/.gitlab-ci.yml index 28db8751a..ca66b9a6f 100644 --- a/src/slice/.gitlab-ci.yml +++ b/src/slice/.gitlab-ci.yml @@ -21,7 +21,7 @@ build slice: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/start.sh b/src/start.sh deleted file mode 100755 index 32a016cc0..000000000 --- a/src/start.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker network create -d bridge teraflowbridge - -docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge monitoring:dockerfile diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index d677cb823..a6459aebc 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -20,7 +20,7 @@ build ecoc22: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" after_script: diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index e9c8c0c03..98caf3b09 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -20,7 +20,7 @@ build ofc22: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" after_script: diff --git a/src/webui/.gitlab-ci.yml b/src/webui/.gitlab-ci.yml index 59d21dfb6..4ef2c847b 100644 --- a/src/webui/.gitlab-ci.yml +++ b/src/webui/.gitlab-ci.yml @@ -21,7 +21,7 @@ build webui: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: diff --git a/src/ztp/.gitlab-ci.yml b/src/ztp/.gitlab-ci.yml index a40a14f49..bc15c9aaf 100644 --- a/src/ztp/.gitlab-ci.yml +++ b/src/ztp/.gitlab-ci.yml @@ -24,7 +24,7 @@ build ztp: - export IMAGE_TAG=$(grep -m1 '' ./src/$IMAGE_NAME/pom.xml | grep -oP '(?<=>).*(?=<)') - echo "IMAGE_TAG=${IMAGE_TAG}" >> ${BUILD_ENV} - cat ${BUILD_ENV} - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target builder + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target builder after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi artifacts: @@ -50,7 +50,7 @@ unit_test ztp: - docker rm ${REPORTS_CONTAINER} || true script: - echo "Running tests for image ${IMAGE_TAG}" - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target unit-test + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target unit-test # Transfer JaCoCo and Surefire reports from within tests image - docker create --name ${REPORTS_CONTAINER} "$IMAGE_NAME:$IMAGE_TAG" - mkdir -p ${REPORTS_PATH} @@ -60,7 +60,7 @@ unit_test ztp: - cat ${REPORTS_PATH}/coverage.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/JaCoCo Coverage Total:/' - docker run -v "$(pwd)/src/${IMAGE_NAME}:/${IMAGE_NAME}" --rm registry.gitlab.com/haynes/jacoco2cobertura:1.0.7 python /opt/cover2cover.py ${IMAGE_NAME}/reports/jacoco.xml ${IMAGE_NAME}/src/main/java > ${REPORTS_PATH}/cobertura.xml # Build final image - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target release + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/src/main/docker/Dockerfile.multistage.jvm ./src/$IMAGE_NAME/ --target release - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" after_script: -- GitLab From 84a84bdd4014191763bf7713ba475cfa01d0b866 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 09:35:48 +0000 Subject: [PATCH 68/73] Manifests: - Disabled policy and ZTP HPAs --- manifests/policyservice.yaml | 38 ++++++++++++++++++------------------ manifests/ztpservice.yaml | 38 ++++++++++++++++++------------------ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/manifests/policyservice.yaml b/manifests/policyservice.yaml index b93eeda03..7ef040e7f 100644 --- a/manifests/policyservice.yaml +++ b/manifests/policyservice.yaml @@ -108,22 +108,22 @@ spec: limits: cpu: 2000m memory: 2048Mi ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: policyservice-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: policyservice - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 +#--- +#apiVersion: autoscaling/v2 +#kind: HorizontalPodAutoscaler +#metadata: +# name: policyservice-hpa +#spec: +# scaleTargetRef: +# apiVersion: apps/v1 +# kind: Deployment +# name: policyservice +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# target: +# type: Utilization +# averageUtilization: 80 diff --git a/manifests/ztpservice.yaml b/manifests/ztpservice.yaml index 323d3c4bc..608bab125 100644 --- a/manifests/ztpservice.yaml +++ b/manifests/ztpservice.yaml @@ -104,22 +104,22 @@ spec: limits: cpu: 2000m memory: 2048Mi ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: ztpservice-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: ztpservice - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 +#--- +#apiVersion: autoscaling/v2 +#kind: HorizontalPodAutoscaler +#metadata: +# name: ztpservice-hpa +#spec: +# scaleTargetRef: +# apiVersion: apps/v1 +# kind: Deployment +# name: ztpservice +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# target: +# type: Utilization +# averageUtilization: 80 -- GitLab From 4c304841d572de627df4f1e7065dd0e6323fa711 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 10:09:38 +0000 Subject: [PATCH 69/73] Updated README.md file --- README.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5bfb6d70d..8825795b8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,25 @@ # ETSI TeraFlowSDN Controller -[ETSI OpenSource Group for TeraFlowSDN](https://tfs.etsi.org/) +The [ETSI Software Development Group TeraFlowSDN (SDG TFS)](https://tfs.etsi.org/) is developing an open source cloud native SDN controller enabling smart connectivity services for future networks beyond 5G. -Former, [Teraflow H2020 project](https://teraflow-h2020.eu/) - Secured autonomic traffic management for a Tera of SDN Flows +The project originated from "[Teraflow H2020 project](https://teraflow-h2020.eu/) - Secured autonomic traffic management for a Tera of SDN Flows", a project funded by the European Union’s Horizon 2020 Research and Innovation programme that finished on 30th June 2023. -Branch "master" : [![pipeline status](https://labs.etsi.org/rep/tfs/controller/badges/master/pipeline.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/master) [![coverage report](https://labs.etsi.org/rep/tfs/controller/badges/master/coverage.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/master) -Branch "develop" : [![pipeline status](https://labs.etsi.org/rep/tfs/controller/badges/develop/pipeline.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/develop) [![coverage report](https://labs.etsi.org/rep/tfs/controller/badges/develop/coverage.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/develop) +## Available branches and releases -# Installation Instructions and Functional Tests -The [TeraFlowSDN Wiki](https://labs.etsi.org/rep/tfs/controller/-/wikis/home) pages include details on using the ETSI TeraFlowSDN release 2.0. +[![Latest Release](https://labs.etsi.org/rep/tfs/controller/-/badges/release.svg)](https://labs.etsi.org/rep/tfs/controller/-/releases) -The documentation, installation instructions, and description of the functional tests defined to enable experimentation with the ETSI TeraFlowSDN Controller can be found in the Wiki pages +- The branch `master` ([![pipeline status](https://labs.etsi.org/rep/tfs/controller/badges/master/pipeline.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/master) [![coverage report](https://labs.etsi.org/rep/tfs/controller/badges/master/coverage.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/master)), points always to the latest stable version of the TeraFlowSDN controller. + +- The branches `release/X.Y.Z`, point to the code for the different release versions indicated in the branch name. + - Code in these branches can be considered stable, and no new features are planned. + - In case of bugs, point releases increasing revision number (Z) might be created. + +- The `develop` ([![pipeline status](https://labs.etsi.org/rep/tfs/controller/badges/develop/pipeline.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/develop) [![coverage report](https://labs.etsi.org/rep/tfs/controller/badges/develop/coverage.svg)](https://labs.etsi.org/rep/tfs/controller/-/commits/develop)) branch is the main development branch and contains the latest contributions. + - **Use it with care! It might not be stable.** + - The latest developments and contributions are added to this branch for testing and validation before reaching a release. + + +## Documentation +The [TeraFlowSDN Wiki](https://labs.etsi.org/rep/tfs/controller/-/wikis/home) pages include the main documentation for the ETSI TeraFlowSDN Controller. +The documentation includes project documentation, installation instructions, functional tests, supported NBIs and SBIs, etc. -- GitLab From 4066f05a22fdba104e85f6cea27cd596e9c2ae49 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 10:15:24 +0000 Subject: [PATCH 70/73] Gitlab CI/CD pipeline - OFC'22 & ECOC'22: - Removed conftest settings as seem to collide with error reporting. - Enforced re-build of docker images --- src/tests/ecoc22/.gitlab-ci.yml | 8 ++-- src/tests/ecoc22/tests/conftest.py | 63 ------------------------------ src/tests/ofc22/.gitlab-ci.yml | 8 ++-- src/tests/ofc22/tests/conftest.py | 63 ------------------------------ 4 files changed, 8 insertions(+), 134 deletions(-) delete mode 100644 src/tests/ecoc22/tests/conftest.py delete mode 100644 src/tests/ofc22/tests/conftest.py diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index a6459aebc..09718b4b9 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -62,10 +62,10 @@ end2end_test ecoc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="YES" - - export TFS_IMAGE_TAG="latest" - - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh diff --git a/src/tests/ecoc22/tests/conftest.py b/src/tests/ecoc22/tests/conftest.py deleted file mode 100644 index 6650864b0..000000000 --- a/src/tests/ecoc22/tests/conftest.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Acknowledgement: https://stackoverflow.com/questions/46766899/pytest-timeout-fail-test-instead-killing-whole-test-run - -import pytest, signal - - -class Termination(SystemExit): - pass - - -class TimeoutExit(BaseException): - pass - - -def _terminate(signum, frame): - raise Termination("Runner is terminated from outside.") - - -def _timeout(signum, frame): - raise TimeoutExit("Runner timeout is reached, runner is terminating.") - - -@pytest.hookimpl -def pytest_addoption(parser): - parser.addoption( - '--timeout', action='store', dest='timeout', type=int, default=None, - help="number of seconds before each test failure") - - -@pytest.hookimpl -def pytest_configure(config): - # Install the signal handlers that we want to process. - signal.signal(signal.SIGTERM, _terminate) - signal.signal(signal.SIGALRM, _timeout) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - - # Set the per-test timeout (an alarm signal). - if item.config.option.timeout is not None: - signal.alarm(item.config.option.timeout) - - try: - # Run the setup, test body, and teardown stages. - yield - finally: - # Disable the alarm when the test passes or fails. - # I.e. when we get into the framework's body. - signal.alarm(0) diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 98caf3b09..013a389bc 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -63,10 +63,10 @@ end2end_test ofc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh - - export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" - - export TFS_SKIP_BUILD="YES" - - export TFS_IMAGE_TAG="latest" - - echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" # Deploy TeraFlowSDN - ./deploy/crdb.sh diff --git a/src/tests/ofc22/tests/conftest.py b/src/tests/ofc22/tests/conftest.py deleted file mode 100644 index 6650864b0..000000000 --- a/src/tests/ofc22/tests/conftest.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Acknowledgement: https://stackoverflow.com/questions/46766899/pytest-timeout-fail-test-instead-killing-whole-test-run - -import pytest, signal - - -class Termination(SystemExit): - pass - - -class TimeoutExit(BaseException): - pass - - -def _terminate(signum, frame): - raise Termination("Runner is terminated from outside.") - - -def _timeout(signum, frame): - raise TimeoutExit("Runner timeout is reached, runner is terminating.") - - -@pytest.hookimpl -def pytest_addoption(parser): - parser.addoption( - '--timeout', action='store', dest='timeout', type=int, default=None, - help="number of seconds before each test failure") - - -@pytest.hookimpl -def pytest_configure(config): - # Install the signal handlers that we want to process. - signal.signal(signal.SIGTERM, _terminate) - signal.signal(signal.SIGALRM, _timeout) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - - # Set the per-test timeout (an alarm signal). - if item.config.option.timeout is not None: - signal.alarm(item.config.option.timeout) - - try: - # Run the setup, test body, and teardown stages. - yield - finally: - # Disable the alarm when the test passes or fails. - # I.e. when we get into the framework's body. - signal.alarm(0) -- GitLab From aad71eb8b5acbebc76f0f1a887b43ab0db31ca08 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 15:42:15 +0000 Subject: [PATCH 71/73] Optical Controller component: - Added CI/CD pipeline descriptor - Homogeneized Dockerfile --- .gitlab-ci.yml | 1 + src/opticalcontroller/.gitlab-ci.yml | 105 +++++++++++++++++++++++++++ src/opticalcontroller/Dockerfile | 40 ++++------ 3 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 src/opticalcontroller/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 064cedd15..2d7c2e21b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ include: - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - local: '/src/opticalattackdetector/.gitlab-ci.yml' - local: '/src/opticalattackmanager/.gitlab-ci.yml' + - local: '/src/opticalcontroller/.gitlab-ci.yml' - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' - local: '/src/forecaster/.gitlab-ci.yml' diff --git a/src/opticalcontroller/.gitlab-ci.yml b/src/opticalcontroller/.gitlab-ci.yml new file mode 100644 index 000000000..4f27f0a42 --- /dev/null +++ b/src/opticalcontroller/.gitlab-ci.yml @@ -0,0 +1,105 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build opticalcontroller: + variables: + IMAGE_NAME: 'opticalcontroller' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +## Apply unit test to the component +#unit_test opticalcontroller: +# variables: +# IMAGE_NAME: 'opticalcontroller' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build opticalcontroller +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker run --name $IMAGE_NAME -d -p 20030:20030 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - sleep 5 +# - docker ps -a +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" +# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy opticalcontroller: +# variables: +# IMAGE_NAME: 'opticalcontroller' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test opticalcontroller +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/opticalcontroller/Dockerfile b/src/opticalcontroller/Dockerfile index c3d886ab5..d54566ccb 100644 --- a/src/opticalcontroller/Dockerfile +++ b/src/opticalcontroller/Dockerfile @@ -16,63 +16,55 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur ENV PYTHONUNBUFFERED=0 # Download the gRPC health probe -# RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ -# wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ -# chmod +x /bin/grpc_health_probe - - - +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe # Get generic Python packages RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel RUN python3 -m pip install --upgrade pip-tools - - +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow COPY common_requirements.in common_requirements.in RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in RUN python3 -m pip install -r common_requirements.txt -RUN mkdir -p /var/teraflow/opticalcontroller - -WORKDIR /var/teraflow/opticalcontroller/common +# Add common files into working directory +WORKDIR /var/teraflow/common COPY src/common/. ./ RUN rm -rf proto - # Create proto sub-folder, copy .proto files, and generate Python code -RUN mkdir -p /var/teraflow/opticalcontroller/common/proto -WORKDIR /var/teraflow/opticalcontroller/common/proto +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto RUN touch __init__.py COPY proto/*.proto ./ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Create component sub-folder, get specific Python packages - - - +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/opticalcontroller WORKDIR /var/teraflow/opticalcontroller COPY src/opticalcontroller/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory -WORKDIR /var/teraflow/ - -COPY src/context/. context/ - +WORKDIR /var/teraflow +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ COPY src/opticalcontroller/. opticalcontroller/ -COPY src/context/. opticalcontroller/context/ # Start the service WORKDIR /var/teraflow/opticalcontroller -- GitLab From e0cda265ce421555729653c8cb801a5bf81efbd7 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 15:42:50 +0000 Subject: [PATCH 72/73] GitLab CI/CD pipeline - ECOC'22: - Added missing device activation test --- .../ecoc22/tests/test_functional_bootstrap.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/tests/ecoc22/tests/test_functional_bootstrap.py b/src/tests/ecoc22/tests/test_functional_bootstrap.py index 60df8d8a2..edafbae31 100644 --- a/src/tests/ecoc22/tests/test_functional_bootstrap.py +++ b/src/tests/ecoc22/tests/test_functional_bootstrap.py @@ -14,7 +14,7 @@ import logging, os, time from common.Constants import DEFAULT_CONTEXT_NAME -from common.proto.context_pb2 import ContextId +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario from common.tools.object_factory.Context import json_context_id from context.client.ContextClient import ContextClient @@ -43,3 +43,25 @@ def test_scenario_bootstrap( response = context_client.GetContext(ADMIN_CONTEXT_ID) assert len(response.service_ids) == 0 assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 + assert num_devices_enabled == num_devices -- GitLab From cef04c4cdc8bddad58a8d922b7fc1a44827de391 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 22 Mar 2024 15:47:06 +0000 Subject: [PATCH 73/73] GitLab CI/CD pipeline - OFC'24: - Added preliminary non-functional test scripts - Renamed scripts and data files - Added python code to test deployments - Added CI/CD pipeline descriptor - Added TFS descriptor file - Added Dockerfile - Added script to deploy node agents --- src/tests/ofc24/.gitlab-ci.yml | 106 +++++++ src/tests/ofc24/Dockerfile | 101 ++++++ src/tests/ofc24/deploy-node-agents.sh | 84 +++++ src/tests/ofc24/deploy_specs.sh | 163 ++++++++++ src/tests/ofc24/descriptors_topology.json | 150 +++++++++ .../ofc24/{plat_r1.xml => platform_r1.xml} | 2 +- .../ofc24/{plat_r2.xml => platform_r2.xml} | 2 +- src/tests/ofc24/platform_t1.xml | 295 +++++++++++++++++ src/tests/ofc24/platform_t2.xml | 295 +++++++++++++++++ src/tests/ofc24/requirements.in | 15 + src/tests/ofc24/startNetconfAgent.sh | 7 + src/tests/ofc24/t1.xml | 298 ------------------ src/tests/ofc24/t2.xml | 298 ------------------ src/tests/ofc24/tests/__init__.py | 14 + .../ofc24/tests/test_functional_bootstrap.py | 67 ++++ .../ofc24/tests/test_functional_cleanup.py | 44 +++ .../tests/test_functional_create_service.py | 102 ++++++ .../tests/test_functional_delete_service.py | 74 +++++ 18 files changed, 1519 insertions(+), 598 deletions(-) create mode 100644 src/tests/ofc24/.gitlab-ci.yml create mode 100644 src/tests/ofc24/Dockerfile create mode 100755 src/tests/ofc24/deploy-node-agents.sh create mode 100755 src/tests/ofc24/deploy_specs.sh create mode 100644 src/tests/ofc24/descriptors_topology.json rename src/tests/ofc24/{plat_r1.xml => platform_r1.xml} (99%) rename src/tests/ofc24/{plat_r2.xml => platform_r2.xml} (99%) create mode 100755 src/tests/ofc24/platform_t1.xml create mode 100755 src/tests/ofc24/platform_t2.xml create mode 100644 src/tests/ofc24/requirements.in create mode 100755 src/tests/ofc24/startNetconfAgent.sh delete mode 100755 src/tests/ofc24/t1.xml delete mode 100755 src/tests/ofc24/t2.xml create mode 100644 src/tests/ofc24/tests/__init__.py create mode 100644 src/tests/ofc24/tests/test_functional_bootstrap.py create mode 100644 src/tests/ofc24/tests/test_functional_cleanup.py create mode 100644 src/tests/ofc24/tests/test_functional_create_service.py create mode 100644 src/tests/ofc24/tests/test_functional_delete_service.py diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml new file mode 100644 index 000000000..0b5593b16 --- /dev/null +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -0,0 +1,106 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build ofc22: + variables: + TEST_NAME: 'ofc22' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test ofc22: + variables: + TEST_NAME: 'ofc22' + stage: end2end_test + # Disable to force running it after all other tasks + #needs: + # - build ofc22 + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + + # Check MicroK8s is ready + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + + # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + + # Deploy TeraFlowSDN + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/qdb.sh + - ./deploy/expose_dashboard.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + + # Wait for Context to be subscribed to NATS + #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + + # Run end-to-end tests + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - > + docker run -t --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/ofc24/Dockerfile b/src/tests/ofc24/Dockerfile new file mode 100644 index 000000000..8efa0c72c --- /dev/null +++ b/src/tests/ofc24/Dockerfile @@ -0,0 +1,101 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/ofc24 +WORKDIR /var/teraflow/tests/ofc24 +COPY src/tests/ofc24/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +#COPY src/monitoring/__init__.py monitoring/__init__.py +#COPY src/monitoring/client/. monitoring/client/ +#COPY src/monitoring/__init__.py monitoring/__init__.py +#COPY src/monitoring/client/. monitoring/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/ofc24/__init__.py ./tests/ofc24/__init__.py +COPY src/tests/ofc24/descriptors_topology.json ./tests/ofc24/descriptors_topology.json +COPY src/tests/ofc24/tests/. ./tests/ofc24/tests/ +COPY src/tests/tools/. ./tests/tools/ + +RUN tee ./run_tests.sh <&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done +while ! docker logs na-t2 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done +while ! docker logs na-r1 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done +while ! docker logs na-r2 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done +sleep 2 +docker ps -a + + +echo +echo "Dump Node Agent status:" +echo "-----------------------" +docker ps -a +#docker logs na-t1 +#docker logs na-t2 +#docker logs na-r1 +#docker logs na-r2 + + +#echo +#echo "Post-test clean-up:" +#echo "-------------------" +#docker rm -f na-t1 na-t2 na-r1 na-r2 +#docker network rm na-br + +echo "Done!" diff --git a/src/tests/ofc24/deploy_specs.sh b/src/tests/ofc24/deploy_specs.sh new file mode 100755 index 000000000..ca5494de2 --- /dev/null +++ b/src/tests/ofc24/deploy_specs.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" + +# Uncomment to activate Monitoring +#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate BGP-LS Speaker +#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" + +# Uncomment to activate Optical Controller +export TFS_COMPONENTS="${TFS_COMPONENTS} opticalcontroller" + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/src/tests/ofc24/descriptors_topology.json b/src/tests/ofc24/descriptors_topology.json new file mode 100644 index 000000000..6ae521c76 --- /dev/null +++ b/src/tests/ofc24/descriptors_topology.json @@ -0,0 +1,150 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "T1"}}, "device_type": "optical-transponder", "device_drivers": [11], + "device_operational_status": 1, + "device_endpoints": [ + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "1"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }} + ], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "2023"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "force_running": false, "hostkey_verify": false, + "look_for_keys": false, "allow_agent": false, "commit_per_rule": false, + "device_params": {"name": "default"}, "manager_params": {"timeout": 120}, + "endpoints": [{"uuid": "1", "type": "optical", "sample_types": [101, 102, 201, 202]}] + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "T2"}}, "device_type": "optical-transponder", "device_drivers": [11], + "device_operational_status": 1, + "device_endpoints": [ + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "6"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }} + ], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "2024"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "force_running": false, "hostkey_verify": false, + "look_for_keys": false, "allow_agent": false, "commit_per_rule": false, + "device_params": {"name": "default"}, "manager_params": {"timeout": 120}, + "endpoints": [{"uuid": "6", "type": "optical", "sample_types": [101, 102, 201, 202]}] + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R1"}}, "device_type": "optical-roadm", "device_drivers": [11], + "device_operational_status": 1, + "device_endpoints": [ + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "3"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "12"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "13"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }} + ], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "2025"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "force_running": false, "hostkey_verify": false, + "look_for_keys": false, "allow_agent": false, "commit_per_rule": false, + "device_params": {"name": "default"}, "manager_params": {"timeout": 120}, + "endpoints": [ + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "2"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "3"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "12"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "13"} + ]} + }}] + } + }, + { + "device_id": {"device_uuid": {"uuid": "R2"}}, "device_type": "optical-roadm", "device_drivers": [11], + "device_operational_status": 1, + "device_endpoints": [ + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "4"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "5"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "14"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }}, + {"endpoint_id": { + "device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "15"}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}} + }} + ], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "2026"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "force_running": false, "hostkey_verify": false, + "look_for_keys": false, "allow_agent": false, "commit_per_rule": false, + "device_params": {"name": "default"}, "manager_params": {"timeout": 120}, + "endpoints": [ + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "4"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "5"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "14"}, + {"sample_types": [101, 102, 201, 202], "type": "optical", "uuid": "15"} + ] + }}} + ]} + } + ], + "links": [ + {"link_id": {"link_uuid": {"uuid": "T1->R1"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "1"}}, + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "12"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "R1->T1"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2"}}, + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "1"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "R1->R2"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "3"}}, + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "14"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "R2->R1"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "4"}}, + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "13"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "T2->R2"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "6"}}, + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "15"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "R2->T2"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "5"}}, + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "6"}} + ]} + ] +} diff --git a/src/tests/ofc24/plat_r1.xml b/src/tests/ofc24/platform_r1.xml similarity index 99% rename from src/tests/ofc24/plat_r1.xml rename to src/tests/ofc24/platform_r1.xml index 47e135c2e..625d7048a 100755 --- a/src/tests/ofc24/plat_r1.xml +++ b/src/tests/ofc24/platform_r1.xml @@ -117,4 +117,4 @@ - \ No newline at end of file + diff --git a/src/tests/ofc24/plat_r2.xml b/src/tests/ofc24/platform_r2.xml similarity index 99% rename from src/tests/ofc24/plat_r2.xml rename to src/tests/ofc24/platform_r2.xml index dfaaf05ad..65f7ee458 100755 --- a/src/tests/ofc24/plat_r2.xml +++ b/src/tests/ofc24/platform_r2.xml @@ -117,4 +117,4 @@ - \ No newline at end of file + diff --git a/src/tests/ofc24/platform_t1.xml b/src/tests/ofc24/platform_t1.xml new file mode 100755 index 000000000..09f316211 --- /dev/null +++ b/src/tests/ofc24/platform_t1.xml @@ -0,0 +1,295 @@ + + + + device + + device + + + MellanoxSwitch + SSSA-CNIT + 1.0.0 + 1.0.0 + 1.0.0 + 610610 + typex:OPERATING_SYSTEM + + + + channel-1 + + channel-1 + + + channel-1 + typex:OPTICAL_CHANNEL + + + + 191600000 + 100 + 0 + transceiver-1 + + + 191600000 + 0 + 0 + transceiver-1 + 1 + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + + + + transceiver-1 + + transceiver-1 + + + transceiver-1 + typex:TRANSCEIVER + + + + true + typex:QSFP56_DD_TYPE1 + typex:ETH_400GBASE_ZR + typex:FEC_AUTO + typex:TYPE_DIGITAL_COHERENT_OPTIC + + + true + typex:QSFP56_DD_TYPE1 + typex:ETH_400GBASE_ZR + typex:FEC_AUTO + typex:TYPE_DIGITAL_COHERENT_OPTIC + Cisco + 400zr-QSFP-DD + 01 + 1567321 + + + + 1 + + 1 + channel-1 + + + + + + + + port-1 + + port-1 + + + port-1 + typex:PORT + + + + channel-1 + + channel-1 + + + channel-1 + + + + + + onos-index + + onos-index + 4 + + + onos-index + 4 + + + + odtn-port-type + + odtn-port-type + line + + + odtn-port-type + line + + + + + + + + + + + + 1 + + 1 + Logical channel 1 + DISABLED + type:PROT_OTN + NONE + + + 1 + Logical channel 1 + DISABLED + type:PROT_OTN + NONE + UP + + + + transceiver-1 + + + transceiver-1 + + + + + test1 + test1 + + + test1 + test1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + 0.0 + 0.0 + 0.0 + 0.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0 + + + + + + 1 + + 1 + Optical channel assigned 100 + 100 + OPTICAL_CHANNEL + channel-1 + + + 1 + Optical channel assigned 100 + 100 + OPTICAL_CHANNEL + channel-1 + + + + + + + + 1 + + 1 + FEC1 + Ericsson + + + + 2 + + 2 + FEC2 + Ericsson + + + + + diff --git a/src/tests/ofc24/platform_t2.xml b/src/tests/ofc24/platform_t2.xml new file mode 100755 index 000000000..03c643c91 --- /dev/null +++ b/src/tests/ofc24/platform_t2.xml @@ -0,0 +1,295 @@ + + + + device + + device + + + MellanoxSwitch + SSSA-CNIT + 1.0.0 + 1.0.0 + 1.0.0 + 610610 + typex:OPERATING_SYSTEM + + + + channel-6 + + channel-6 + + + channel-6 + typex:OPTICAL_CHANNEL + + + + 191600000 + 100 + 0 + transceiver-6 + + + 191600000 + 0 + 0 + transceiver-6 + 1 + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + + + + + + transceiver-6 + + transceiver-6 + + + transceiver-6 + typex:TRANSCEIVER + + + + true + typex:QSFP56_DD_TYPE1 + typex:ETH_400GBASE_ZR + typex:FEC_AUTO + typex:TYPE_DIGITAL_COHERENT_OPTIC + + + true + typex:QSFP56_DD_TYPE1 + typex:ETH_400GBASE_ZR + typex:FEC_AUTO + typex:TYPE_DIGITAL_COHERENT_OPTIC + Cisco + 400zr-QSFP-DD + 01 + 1567321 + + + + 1 + + 1 + channel-6 + + + + + + + + port-6 + + port-6 + + + port-6 + typex:PORT + + + + channel-6 + + channel-6 + + + channel-6 + + + + + + onos-index + + onos-index + 4 + + + onos-index + 4 + + + + odtn-port-type + + odtn-port-type + line + + + odtn-port-type + line + + + + + + + + + + + + 4 + + 4 + Logical channel 4 + DISABLED + type:PROT_OTN + NONE + + + 4 + Logical channel 4 + DISABLED + type:PROT_OTN + NONE + UP + + + + transceiver-6 + + + transceiver-6 + + + + + test1 + test1 + + + test1 + test1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + 0.0 + 0.0 + 0.0 + 0.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0 + + + + + + 1 + + 1 + Optical channel assigned 100 + 100 + OPTICAL_CHANNEL + channel-6 + + + 1 + Optical channel assigned 100 + 100 + OPTICAL_CHANNEL + channel-6 + + + + + + + + 1 + + 1 + FEC1 + Ericsson + + + + 2 + + 2 + FEC2 + Ericsson + + + + + diff --git a/src/tests/ofc24/requirements.in b/src/tests/ofc24/requirements.in new file mode 100644 index 000000000..30b11e653 --- /dev/null +++ b/src/tests/ofc24/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +requests==2.27.* diff --git a/src/tests/ofc24/startNetconfAgent.sh b/src/tests/ofc24/startNetconfAgent.sh new file mode 100755 index 000000000..10b721883 --- /dev/null +++ b/src/tests/ofc24/startNetconfAgent.sh @@ -0,0 +1,7 @@ +#!/bin/bash +make clean +make all +#make init +cp init_openconfig-platform.xml confd-cdb/ +#cp init_flex-scale-mg-on.xml confd-cdb/ +make start2 diff --git a/src/tests/ofc24/t1.xml b/src/tests/ofc24/t1.xml deleted file mode 100755 index 712615df8..000000000 --- a/src/tests/ofc24/t1.xml +++ /dev/null @@ -1,298 +0,0 @@ - - - - device - - device - - - MellanoxSwitch - SSSA-CNIT - 1.0.0 - 1.0.0 - 1.0.0 - 610610 - typex:OPERATING_SYSTEM - - - - channel-1 - - channel-1 - - - channel-1 - typex:OPTICAL_CHANNEL - - - - 191600000 - 100 - 0 - transceiver-1 - - - 191600000 - 0 - 0 - transceiver-1 - 1 - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - - - - transceiver-1 - - transceiver-1 - - - transceiver-1 - typex:TRANSCEIVER - - - - true - typex:QSFP56_DD_TYPE1 - typex:ETH_400GBASE_ZR - typex:FEC_AUTO - typex:TYPE_DIGITAL_COHERENT_OPTIC - - - true - typex:QSFP56_DD_TYPE1 - typex:ETH_400GBASE_ZR - typex:FEC_AUTO - typex:TYPE_DIGITAL_COHERENT_OPTIC - Cisco - 400zr-QSFP-DD - 01 - 1567321 - - - - 1 - - 1 - channel-1 - - - - - - - - port-1 - - port-1 - - - port-1 - typex:PORT - - - - channel-1 - - channel-1 - - - channel-1 - - - - - - onos-index - - onos-index - 4 - - - onos-index - 4 - - - - odtn-port-type - - odtn-port-type - line - - - odtn-port-type - line - - - - - - - - - - - - - - 1 - - 1 - Logical channel 1 - DISABLED - type:PROT_OTN - NONE - - - 1 - Logical channel 1 - DISABLED - type:PROT_OTN - NONE - UP - - - - transceiver-1 - - - transceiver-1 - - - - - test1 - test1 - - - test1 - test1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - 0.0 - 0.0 - 0.0 - 0.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0 - - - - - - 1 - - 1 - Optical channel assigned 100 - 100 - OPTICAL_CHANNEL - channel-1 - - - 1 - Optical channel assigned 100 - 100 - OPTICAL_CHANNEL - channel-1 - - - - - - - - 1 - - 1 - FEC1 - Ericsson - - - - 2 - - 2 - FEC2 - Ericsson - - - - - - diff --git a/src/tests/ofc24/t2.xml b/src/tests/ofc24/t2.xml deleted file mode 100755 index 3a35e7e87..000000000 --- a/src/tests/ofc24/t2.xml +++ /dev/null @@ -1,298 +0,0 @@ - - - - device - - device - - - MellanoxSwitch - SSSA-CNIT - 1.0.0 - 1.0.0 - 1.0.0 - 610610 - typex:OPERATING_SYSTEM - - - - channel-6 - - channel-6 - - - channel-6 - typex:OPTICAL_CHANNEL - - - - 191600000 - 100 - 0 - transceiver-6 - - - 191600000 - 0 - 0 - transceiver-6 - 1 - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - - - 0 - 0 - 0 - 0 - 0 - - - - - - transceiver-6 - - transceiver-6 - - - transceiver-6 - typex:TRANSCEIVER - - - - true - typex:QSFP56_DD_TYPE1 - typex:ETH_400GBASE_ZR - typex:FEC_AUTO - typex:TYPE_DIGITAL_COHERENT_OPTIC - - - true - typex:QSFP56_DD_TYPE1 - typex:ETH_400GBASE_ZR - typex:FEC_AUTO - typex:TYPE_DIGITAL_COHERENT_OPTIC - Cisco - 400zr-QSFP-DD - 01 - 1567321 - - - - 1 - - 1 - channel-6 - - - - - - - - port-6 - - port-6 - - - port-6 - typex:PORT - - - - channel-6 - - channel-6 - - - channel-6 - - - - - - onos-index - - onos-index - 4 - - - onos-index - 4 - - - - odtn-port-type - - odtn-port-type - line - - - odtn-port-type - line - - - - - - - - - - - - - - 4 - - 4 - Logical channel 4 - DISABLED - type:PROT_OTN - NONE - - - 4 - Logical channel 4 - DISABLED - type:PROT_OTN - NONE - UP - - - - transceiver-6 - - - transceiver-6 - - - - - test1 - test1 - - - test1 - test1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - 0.0 - 0.0 - 0.0 - 0.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0 - - - - - - 1 - - 1 - Optical channel assigned 100 - 100 - OPTICAL_CHANNEL - channel-6 - - - 1 - Optical channel assigned 100 - 100 - OPTICAL_CHANNEL - channel-6 - - - - - - - - 1 - - 1 - FEC1 - Ericsson - - - - 2 - - 2 - FEC2 - Ericsson - - - - - - diff --git a/src/tests/ofc24/tests/__init__.py b/src/tests/ofc24/tests/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/tests/ofc24/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/ofc24/tests/test_functional_bootstrap.py b/src/tests/ofc24/tests/test_functional_bootstrap.py new file mode 100644 index 000000000..bc648d16d --- /dev/null +++ b/src/tests/ofc24/tests/test_functional_bootstrap.py @@ -0,0 +1,67 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_bootstrap( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 + assert num_devices_enabled == num_devices diff --git a/src/tests/ofc24/tests/test_functional_cleanup.py b/src/tests/ofc24/tests/test_functional_cleanup.py new file mode 100644 index 000000000..5f1ce23f1 --- /dev/null +++ b/src/tests/ofc24/tests/test_functional_cleanup.py @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.tools.descriptor.Loader import DescriptorLoader, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_cleanup( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) diff --git a/src/tests/ofc24/tests/test_functional_create_service.py b/src/tests/ofc24/tests/test_functional_create_service.py new file mode 100644 index 000000000..74c74483e --- /dev/null +++ b/src/tests/ofc24/tests/test_functional_create_service.py @@ -0,0 +1,102 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, random +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, Empty, ServiceTypeEnum +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.descriptor.Loader import DescriptorLoader +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from monitoring.client.MonitoringClient import MonitoringClient +from tests.Fixtures import context_client, monitoring_client # pylint: disable=unused-import +from tests.tools.mock_osm.MockOSM import MockOSM +from .Fixtures import osm_wim # pylint: disable=unused-import +from .Objects import WIM_SERVICE_CONNECTION_POINTS, WIM_SERVICE_TYPE + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_creation(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader(descriptors_file=DESCRIPTOR_FILE, context_client=context_client) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Create Connectivity Service + service_uuid = osm_wim.create_connectivity_service(WIM_SERVICE_TYPE, WIM_SERVICE_CONNECTION_POINTS) + osm_wim.get_connectivity_service_status(service_uuid) + + # Ensure slices and services are created + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.info('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 1 # OSM slice + + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.info('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 2 # 1xL3NM + 1xTAPI + + for service in response.services: + service_id = service.service_id + response = context_client.ListConnections(service_id) + LOGGER.info(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_L3NM: + assert len(response.connections) == 1 # 1 connection per service + elif service.service_type == ServiceTypeEnum.SERVICETYPE_TAPI_CONNECTIVITY_SERVICE: + assert len(response.connections) == 1 # 1 connection per service + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) + + +def test_scenario_kpi_values_created( + monitoring_client: MonitoringClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that KPI values have been inserted into the monitoring database. + We short k KPI descriptors to test. + """ + response = monitoring_client.GetKpiDescriptorList(Empty()) + kpi_descriptors = random.choices(response.kpi_descriptor_list, k=2) + + for kpi_descriptor in kpi_descriptors: + MSG = 'KPI(kpi_uuid={:s}, device_uuid={:s}, endpoint_uuid={:s}, service_uuid={:s}, kpi_sample_type={:s})...' + LOGGER.info(MSG.format( + str(kpi_descriptor.kpi_id.kpi_id.uuid), str(kpi_descriptor.device_id.device_uuid.uuid), + str(kpi_descriptor.endpoint_id.endpoint_uuid.uuid), str(kpi_descriptor.service_id.service_uuid.uuid), + str(KpiSampleType.Name(kpi_descriptor.kpi_sample_type)))) + response = monitoring_client.GetInstantKpi(kpi_descriptor.kpi_id) + kpi_uuid = response.kpi_id.kpi_id.uuid + assert kpi_uuid == kpi_descriptor.kpi_id.kpi_id.uuid + kpi_value_type = response.kpi_value.WhichOneof('value') + if kpi_value_type is None: + MSG = ' KPI({:s}): No instant value found' + LOGGER.warning(MSG.format(str(kpi_uuid))) + else: + kpi_timestamp = response.timestamp.timestamp + assert kpi_timestamp > 0 + assert kpi_value_type == 'floatVal' + kpi_value = getattr(response.kpi_value, kpi_value_type) + MSG = ' KPI({:s}): timestamp={:s} value_type={:s} value={:s}' + LOGGER.info(MSG.format(str(kpi_uuid), str(kpi_timestamp), str(kpi_value_type), str(kpi_value))) diff --git a/src/tests/ofc24/tests/test_functional_delete_service.py b/src/tests/ofc24/tests/test_functional_delete_service.py new file mode 100644 index 000000000..daff29064 --- /dev/null +++ b/src/tests/ofc24/tests/test_functional_delete_service.py @@ -0,0 +1,74 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceTypeEnum +from common.tools.descriptor.Loader import DescriptorLoader +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from tests.Fixtures import context_client # pylint: disable=unused-import +from tests.tools.mock_osm.MockOSM import MockOSM +from .Fixtures import osm_wim # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors_emulated.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_removal(context_client : ContextClient, osm_wim : MockOSM): # pylint: disable=redefined-outer-name + # Ensure slices and services are created + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.info('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 1 # OSM slice + + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.info('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 2 # 1xL3NM + 1xTAPI + + service_uuids = set() + for service in response.services: + service_id = service.service_id + response = context_client.ListConnections(service_id) + LOGGER.info(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_L3NM: + assert len(response.connections) == 1 # 1 connection per service + service_uuid = service_id.service_uuid.uuid + service_uuids.add(service_uuid) + osm_wim.conn_info[service_uuid] = {} + elif service.service_type == ServiceTypeEnum.SERVICETYPE_TAPI_CONNECTIVITY_SERVICE: + assert len(response.connections) == 1 # 1 connection per service + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) + + # Identify service to delete + assert len(service_uuids) == 1 # assume a single L3NM service has been created + service_uuid = set(service_uuids).pop() + + # Delete Connectivity Service + osm_wim.delete_connectivity_service(service_uuid) + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader(descriptors_file=DESCRIPTOR_FILE, context_client=context_client) + descriptor_loader.validate() -- GitLab