Commit 14af1bb3 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

CI end-to-end tests: 

- Factorized gitlab ci runner cleanup
- Added recreation of K8s storage and registry to reduce disk space consumption
parent 12b4483d

scripts/ci_cleanup_gitlab_runner.sh

0 → 100755
+38 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2026 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



set -euo pipefail



CI_CLEANUP_CONTAINERLAB="${CI_CLEANUP_CONTAINERLAB:-yes}"

CI_DOCKER_BUILD_CACHE_PRUNE="${CI_DOCKER_BUILD_CACHE_PRUNE:-yes}"



if [[ -f scripts/prune_old_mr_docker_images.sh ]]; then

    bash scripts/prune_old_mr_docker_images.sh

fi



if [[ "${CI_CLEANUP_CONTAINERLAB}" == "yes" ]] && command -v containerlab >/dev/null 2>&1; then

    containerlab destroy --all --cleanup || true

fi



docker ps --all --quiet | xargs --no-run-if-empty docker stop

docker container prune --force

docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

docker image prune --force

docker network prune --force

docker volume prune --all --force



if [[ "${CI_DOCKER_BUILD_CACHE_PRUNE}" == "yes" ]]; then

    docker buildx prune --force

fi

scripts/ci_prepare_microk8s_storage_registry.sh

0 → 100755
+142 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2026 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



set -euo pipefail



MICROK8S_CMD="${MICROK8S_CMD:-microk8s}"

KUBECTL_CMD="${KUBECTL_CMD:-kubectl}"

HELM_CMD="${HELM_CMD:-helm3}"

CI_RESET_MICROK8S_STORAGE_REGISTRY="${CI_RESET_MICROK8S_STORAGE_REGISTRY:-yes}"

CI_K8S_NAMESPACES_PATTERN="${CI_K8S_NAMESPACES_PATTERN:-^(tfs|crdb|qdb|kafka|nats)}"

CI_MICROK8S_WAIT_ATTEMPTS="${CI_MICROK8S_WAIT_ATTEMPTS:-30}"

CI_MICROK8S_WAIT_SLEEP="${CI_MICROK8S_WAIT_SLEEP:-2}"

CI_MICROK8S_ADDON_WAIT_TIMEOUT="${CI_MICROK8S_ADDON_WAIT_TIMEOUT:-180s}"



wait_for_kubectl() {

    local counter=0

    until ${KUBECTL_CMD} get pods --all-namespaces >/dev/null 2>&1; do

        printf "."

        sleep "${CI_MICROK8S_WAIT_SLEEP}"

        counter=$((counter + 1))

        if [[ "${counter}" -ge "${CI_MICROK8S_WAIT_ATTEMPTS}" ]]; then

            echo "Kubernetes API did not become ready in time." >&2

            return 1

        fi

    done

    echo

}



delete_pods_by_phase() {

    local phase=$1

    ${KUBECTL_CMD} get pods --all-namespaces --no-headers --field-selector=status.phase="${phase}" \

        -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |

        xargs --no-run-if-empty --max-args=2 ${KUBECTL_CMD} delete pod --namespace || true

}



wait_for_addon_pods() {

    local namespace=$1

    local label=$2

    if ${KUBECTL_CMD} get namespace "${namespace}" >/dev/null 2>&1; then

        ${KUBECTL_CMD} wait --namespace "${namespace}" --for=condition=Ready pod \

            --selector "${label}" --timeout="${CI_MICROK8S_ADDON_WAIT_TIMEOUT}"

    fi

}



wait_for_storage_ready() {

    ${KUBECTL_CMD} rollout status deployment/hostpath-provisioner --namespace kube-system \

        --timeout="${CI_MICROK8S_ADDON_WAIT_TIMEOUT}" && return 0

    wait_for_addon_pods kube-system name=hostpath-provisioner && return 0

    wait_for_addon_pods kube-system app=hostpath-provisioner && return 0

    wait_for_addon_pods kube-system k8s-app=hostpath-provisioner && return 0

    echo "MicroK8s hostpath storage did not become ready." >&2

    return 1

}



wait_for_registry_ready() {

    ${KUBECTL_CMD} rollout status deployment/registry --namespace container-registry \

        --timeout="${CI_MICROK8S_ADDON_WAIT_TIMEOUT}" && return 0

    wait_for_addon_pods container-registry app=registry && return 0

    echo "MicroK8s registry did not become ready." >&2

    return 1

}



echo "Checking MicroK8s readiness..."

${MICROK8S_CMD} status --wait-ready

wait_for_kubectl

${KUBECTL_CMD} get pods --all-namespaces



echo "Deleting old TeraFlowSDN-related namespaces..."

K8S_NAMESPACES="$(${KUBECTL_CMD} get namespace -o jsonpath='{.items[*].metadata.name}')"

echo "K8S_NAMESPACES=${K8S_NAMESPACES}"



OLD_NATS_NAMESPACES="$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats' || true)"

echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}"

for ns in ${OLD_NATS_NAMESPACES}; do

    if [[ "${ns}" == nats* ]]; then

        if ${HELM_CMD} status "${ns}" --namespace "${ns}" >/dev/null 2>&1; then

            ${HELM_CMD} uninstall "${ns}" --namespace "${ns}"

        else

            echo "Release '${ns}' not found, skipping..."

        fi

    fi

done



OLD_NAMESPACES="$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E "${CI_K8S_NAMESPACES_PATTERN}" || true)"

echo "OLD_NAMESPACES=${OLD_NAMESPACES}"

if [[ -n "${OLD_NAMESPACES}" ]]; then

    ${KUBECTL_CMD} delete namespace ${OLD_NAMESPACES} --ignore-not-found || true

    for ns in ${OLD_NAMESPACES}; do

        for _ in $(seq 1 "${CI_MICROK8S_WAIT_ATTEMPTS}"); do

            ${KUBECTL_CMD} get namespace "${ns}" >/dev/null 2>&1 || break

            sleep "${CI_MICROK8S_WAIT_SLEEP}"

        done

    done

fi



echo "Deleting completed and failed Kubernetes pods..."

delete_pods_by_phase Failed

delete_pods_by_phase Succeeded



if [[ "${CI_RESET_MICROK8S_STORAGE_REGISTRY}" != "yes" ]]; then

    echo "Skipping MicroK8s storage/registry reset."

    exit 0

fi



echo "Disabling MicroK8s registry and storage addons..."

${MICROK8S_CMD} disable registry || true

${MICROK8S_CMD} disable hostpath-storage --destroy-storage || \

    ${MICROK8S_CMD} disable storage --destroy-storage || \

    ${MICROK8S_CMD} disable hostpath-storage || \

    ${MICROK8S_CMD} disable storage || true



echo "Deleting leftover hostpath provisioner pods and jobs..."

${KUBECTL_CMD} delete pods --namespace kube-system --selector name=hostpath-provisioner --ignore-not-found || true

${KUBECTL_CMD} delete pods --namespace kube-system --selector app=hostpath-provisioner --ignore-not-found || true

${KUBECTL_CMD} delete pods --namespace kube-system --selector k8s-app=hostpath-provisioner --ignore-not-found || true

${KUBECTL_CMD} get jobs --all-namespaces --no-headers \

    -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |

    grep -E 'hostpath|provision' |

    xargs --no-run-if-empty --max-args=2 ${KUBECTL_CMD} delete job --namespace || true



echo "Re-enabling MicroK8s storage and registry addons..."

${MICROK8S_CMD} enable hostpath-storage || ${MICROK8S_CMD} enable storage

${MICROK8S_CMD} enable registry



echo "Waiting for storage and registry pods..."

${MICROK8S_CMD} status --wait-ready

wait_for_kubectl

wait_for_storage_ready

wait_for_registry_ready

${KUBECTL_CMD} get pods --all-namespaces

src/tests/acl_end2end/.gitlab-ci.yml

+5 −76
Original line number Diff line number Diff line
@@ -47,60 +47,7 @@ end2end_test acl_end2end: 
  #needs:

  #  - build acl_end2end

  before_script:

    - bash scripts/prune_old_mr_docker_images.sh

    # Cleanup old ContainerLab scenarios

    - containerlab destroy --all --cleanup || true



    # Do Docker cleanup

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker image prune --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker buildx prune --force



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces



    # Always delete Kubernetes namespaces

    - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}')

    - echo "K8S_NAMESPACES=${K8S_NAMESPACES}"



    - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats')

    - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}"

    - >

      for ns in ${OLD_NATS_NAMESPACES}; do

        if [[ "$ns" == nats* ]]; then

          if helm3 status "$ns" &>/dev/null; then

            helm3 uninstall "$ns" -n "$ns"

          else

            echo "Release '$ns' not found, skipping..."

          fi

        fi

      done

    - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)')

    - echo "OLD_NAMESPACES=${OLD_NAMESPACES}"

    - kubectl delete namespace ${OLD_NAMESPACES} || true



    # Clean-up Kubernetes Failed pods

    - >

      kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed

      -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |

      xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace

    - bash scripts/ci_cleanup_gitlab_runner.sh



    # Login Docker repository

    - bash scripts/dockerhub_login.sh
@@ -110,22 +57,6 @@ end2end_test acl_end2end: 
    # Download Docker image to run the test

    - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest"



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces



    # Deploy ContainerLab Scenario

    - RUNNER_PATH=`pwd`

    #- cd $PWD/src/tests/${TEST_NAME}
@@ -143,6 +74,9 @@ end2end_test acl_end2end: 
    # Dump configuration of the firewall (before any configuration)

    - containerlab exec --name acl_end2end --label clab-node-name=firewall --cmd "Cli --command \"enable"$'\n'$"show running-config\""



    # Prepare Kubernetes storage and registry

    - bash scripts/ci_prepare_microk8s_storage_registry.sh



    # Configure TeraFlowSDN deployment

    # Uncomment if DEBUG log level is needed for the components

    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml
@@ -279,12 +213,7 @@ end2end_test acl_end2end: 
    - sudo rm -rf clab-acl_end2end/ .acl_end2end.clab.yml.bak || true

    - cd $RUNNER_PATH

    - kubectl delete namespaces tfs || true

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker image prune --force

    - bash scripts/ci_cleanup_gitlab_runner.sh



  #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'

  rules:

src/tests/ecoc22/.gitlab-ci.yml

+4 −72
Original line number Diff line number Diff line
@@ -47,57 +47,7 @@ end2end_test ecoc22: 
  #needs:

  #  - build ecoc22

  before_script:

    - bash scripts/prune_old_mr_docker_images.sh

    # Do Docker cleanup

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker image prune --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker buildx prune --force



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces



    # Always delete Kubernetes namespaces

    - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}')

    - echo "K8S_NAMESPACES=${K8S_NAMESPACES}"



    - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats')

    - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}"

    - >

      for ns in ${OLD_NATS_NAMESPACES}; do

        if [[ "$ns" == nats* ]]; then

          if helm3 status "$ns" &>/dev/null; then

            helm3 uninstall "$ns" -n "$ns"

          else

            echo "Release '$ns' not found, skipping..."

          fi

        fi

      done

    - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)')

    - echo "OLD_NAMESPACES=${OLD_NAMESPACES}"

    - kubectl delete namespace ${OLD_NAMESPACES} || true



    # Clean-up Kubernetes Failed pods

    - >

      kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed

      -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |

      xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace

    - bash scripts/ci_cleanup_gitlab_runner.sh



    # Login Docker repository

    - bash scripts/dockerhub_login.sh
@@ -107,21 +57,8 @@ end2end_test ecoc22: 
    # Download Docker image to run the test

    - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest"



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces

    # Prepare Kubernetes storage and registry

    - bash scripts/ci_prepare_microk8s_storage_registry.sh



    # Configure TeraFlowSDN deployment

    # Uncomment if DEBUG log level is needed for the components
@@ -172,12 +109,7 @@ end2end_test ecoc22: 


    # Clean up

    - kubectl delete namespaces tfs || true

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker image prune --force

    - bash scripts/ci_cleanup_gitlab_runner.sh



  #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'

  rules:

src/tests/eucnc24/.gitlab-ci.yml

+5 −76
Original line number Diff line number Diff line
@@ -47,60 +47,7 @@ end2end_test eucnc24: 
  #needs:

  #  - build eucnc24

  before_script:

    - bash scripts/prune_old_mr_docker_images.sh

    # Cleanup old ContainerLab scenarios

    - containerlab destroy --all --cleanup || true



    # Do Docker cleanup

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker image prune --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker buildx prune --force



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces



    # Always delete Kubernetes namespaces

    - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}')

    - echo "K8S_NAMESPACES=${K8S_NAMESPACES}"



    - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats')

    - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}"

    - >

      for ns in ${OLD_NATS_NAMESPACES}; do

        if [[ "$ns" == nats* ]]; then

          if helm3 status "$ns" &>/dev/null; then

            helm3 uninstall "$ns" -n "$ns"

          else

            echo "Release '$ns' not found, skipping..."

          fi

        fi

      done

    - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)')

    - echo "OLD_NAMESPACES=${OLD_NAMESPACES}"

    - kubectl delete namespace ${OLD_NAMESPACES} || true



    # Clean-up Kubernetes Failed pods

    - >

      kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed

      -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |

      xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace

    - bash scripts/ci_cleanup_gitlab_runner.sh



    # Login Docker repository

    - bash scripts/dockerhub_login.sh
@@ -110,22 +57,6 @@ end2end_test eucnc24: 
    # Download Docker image to run the test

    - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest"



    # Check MicroK8s is ready

    - microk8s status --wait-ready

    - LOOP_MAX_ATTEMPTS=10

    - LOOP_COUNTER=0

    - >

      while ! kubectl get pods --all-namespaces &> /dev/null; do

        printf "%c" "."

        sleep 1

        LOOP_COUNTER=$((LOOP_COUNTER + 1))

        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then

          echo "Max attempts reached, exiting the loop."

          exit 1

        fi

      done

    - kubectl get pods --all-namespaces



    # Deploy ContainerLab Scenario

    - RUNNER_PATH=`pwd`

    #- cd $PWD/src/tests/${TEST_NAME}
@@ -145,6 +76,9 @@ end2end_test eucnc24: 
    - containerlab exec --name ${TEST_NAME} --label clab-node-name=r2 --cmd "Cli --command \"enable"$'\n'$"show running-config\""

    - containerlab exec --name ${TEST_NAME} --label clab-node-name=r3 --cmd "Cli --command \"enable"$'\n'$"show running-config\""



    # Prepare Kubernetes storage and registry

    - bash scripts/ci_prepare_microk8s_storage_registry.sh



    # Configure TeraFlowSDN deployment

    # Uncomment if DEBUG log level is needed for the components

    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml
@@ -312,12 +246,7 @@ end2end_test eucnc24: 
    - sudo rm -rf clab-${TEST_NAME}/ .${TEST_NAME}.clab.yml.bak || true

    - cd $RUNNER_PATH

    - kubectl delete namespaces tfs || true

    - docker ps --all --quiet | xargs --no-run-if-empty docker stop

    - docker container prune --force

    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force

    - docker network prune --force

    - docker volume prune --all --force

    - docker image prune --force

    - bash scripts/ci_cleanup_gitlab_runner.sh



  #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'

  rules: