Commit 08f618d8 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

OFC25 end-to-end integration test:

- Updated (partial) CI/CD pipeline
parent 83014c9f
Loading
Loading
Loading
Loading
+178 −107
Original line number Diff line number Diff line
@@ -46,27 +46,122 @@ end2end_test ofc25:
  #needs:
  #  - build ofc25
  before_script:
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - docker network rm -f na-br
    # Do Docker cleanup
    - docker ps --all --quiet | xargs --no-run-if-empty docker stop
    - docker container prune --force
    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force
    - docker image prune --force
    - docker network prune --force
    - docker volume prune --all --force
    - docker buildx prune --force

    # Check MicroK8s is ready
    - microk8s status --wait-ready
    - LOOP_MAX_ATTEMPTS=10
    - LOOP_COUNTER=0
    - >
      while ! kubectl get pods --all-namespaces &> /dev/null; do
        printf "%c" "."
        sleep 1
        LOOP_COUNTER=$((LOOP_COUNTER + 1))
        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then
          echo "Max attempts reached, exiting the loop."
          exit 1
        fi
      done
    - kubectl get pods --all-namespaces

    # Delete secondary ingress controllers
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-opt.yaml --ignore-not-found
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-ip.yaml --ignore-not-found
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-e2e.yaml --ignore-not-found

    # Always delete Kubernetes namespaces
    - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}')
    - echo "K8S_NAMESPACES=${K8S_NAMESPACES}"

    - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats')
    - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}"
    - >
      for ns in ${OLD_NATS_NAMESPACES}; do
        if [[ "$ns" == nats* ]]; then
          if helm3 status "$ns" &>/dev/null; then
            helm3 uninstall "$ns" -n "$ns"
          else
            echo "Release '$ns' not found, skipping..."
          fi
        fi
      done
    - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)')
    - echo "OLD_NAMESPACES=${OLD_NAMESPACES}"
    - kubectl delete namespace ${OLD_NAMESPACES} || true

    # Clean-up Kubernetes Failed pods
    - >
      kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed
      -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |
      xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace

    # Login Docker repository
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

    - helm3 uninstall --namespace nats-e2e nats-e2e 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats-ip nats-ip 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats-opt nats-opt 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats nats 2>/dev/null  || echo "Namespace not found"
    - kubectl delete namespaces tfs tfs-ip tfs-opt tfs-e2e qdb qdb-e2e qdb-opt qdb-ip --ignore-not-found
    - kubectl delete namespaces nats nats-ip nats-opt nats-e2e --ignore-not-found
    - echo "HOLA"
  script:
    # Download Docker image to run the test
    - echo "Que tal"
    - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest"

    - docker pull asgamb1/oc23bgp.img:latest
    - docker pull asgamb1/flexscale-node.img:latest

    # Check MicroK8s is ready
    - microk8s status --wait-ready
    - LOOP_MAX_ATTEMPTS=10
    - LOOP_COUNTER=0
    - >
      while ! kubectl get pods --all-namespaces &> /dev/null; do
        printf "%c" "."
        sleep 1
        LOOP_COUNTER=$((LOOP_COUNTER + 1))
        if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then
          echo "Max attempts reached, exiting the loop."
          exit 1
        fi
      done
    - kubectl get pods --all-namespaces

    # Deploy Optical Device Node Agents
    - >
      docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254
      --ip-range=172.254.253.0/24 na-br
    - >
      docker run -dit --init --name na-t1 --network=na-br --ip 172.254.253.101 --publish 2022
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/startNetconfAgent-tp.sh:/confd/examples.confd/OC23/startNetconfAgent.sh"
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_t1.xml:/confd/examples.confd/OC23/platform.xml"
      asgamb1/oc23bgp.img:latest /confd/examples.confd/OC23/startNetconfAgent.sh
    - >
      docker run -dit --init --name na-t2 --network=na-br --ip 172.254.253.102 --publish 2022
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/startNetconfAgent-tp.sh:/confd/examples.confd/OC23/startNetconfAgent.sh"
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_t2.xml:/confd/examples.confd/OC23/platform.xml"
      asgamb1/oc23bgp.img:latest /confd/examples.confd/OC23/startNetconfAgent.sh
    - >
      docker run -dit --init --name na-r1 --network=na-br --ip 172.254.253.201 --publish 2022
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/startNetconfAgent-mg-on.sh:/confd/examples.confd/OC23/startNetconfAgent.sh"
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_r1.xml:/confd/examples.confd/OC23/platform.xml"
      asgamb1/flexscale-node.img:latest /confd/examples.confd/OC23/startNetconfAgent.sh
    - >
      docker run -dit --init --name na-r2 --network=na-br --ip 172.254.253.202 --publish 2022
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/startNetconfAgent-mg-on.sh:/confd/examples.confd/OC23/startNetconfAgent.sh"
      --volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_r2.xml:/confd/examples.confd/OC23/platform.xml"
      asgamb1/flexscale-node.img:latest /confd/examples.confd/OC23/startNetconfAgent.sh


    # Wait for initialization of Optical Device Node Agents
    - sleep 3
    - docker ps -a
    - while ! docker logs na-t1 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done
    - while ! docker logs na-t2 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done
    - while ! docker logs na-r1 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done
    - while ! docker logs na-r2 2>&1 | grep -q '*** ConfD OpenConfig NETCONF agent ***'; do sleep 1; done
    - sleep 3
    - docker ps -a


    # Configure TeraFlowSDN deployment
@@ -77,21 +172,16 @@ end2end_test ofc25:
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/opticalcontrollerservice.yaml
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/e2eorchestratorservice.yaml
    #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/vntmservice.yaml



    # Deploy Optical TeraFlowSDN
    # ===== Deploy Optical TeraFlowSDN ==================================================
    - source src/tests/${TEST_NAME}/deploy_specs_opt.sh


    # Delete secondary ingress controllers
    - kubectl delete -f src/tests/ofc25/nginx-ingress-controller-opt.yaml --ignore-not-found
    # Create secondary ingress controllers
    - kubectl apply -f src/tests/ofc25/nginx-ingress-controller-opt.yaml
    # Deploy TFS for OPT
    - source src/tests/ofc25/deploy_specs_opt.sh
    - kubectl apply -f src/tests/${TEST_NAME}/nginx-ingress-controller-opt.yaml

    # Change the name for the database
    - cp manifests/contextservice.yaml manifests/contextservice.yaml.bak
@@ -100,145 +190,126 @@ end2end_test ofc25:

    - ./deploy/crdb.sh
    - ./deploy/nats.sh
    - ./deploy/kafka.sh
    # - ./deploy/qdb.sh


    - ./deploy/expose_dashboard.sh
    # - ./deploy/expose_dashboard.sh
    - ./deploy/tfs.sh
    - ./deploy/show.sh

    - mv manifests/contextservice.yaml.bak manifests/contextservice.yaml

    - mv tfs_runtime_env_vars.sh tfs_runtime_env_vars_opt.sh

    # - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_opt.sh



    # Deploy IP TeraFlowSDN
    # Delete secondary ingress controllers
    - kubectl delete -f src/tests/ofc25/nginx-ingress-controller-ip.yaml --ignore-not-found
    # ===== Deploy Packet TeraFlowSDN ===================================================
    - source src/tests/${TEST_NAME}/deploy_specs_ip.sh

    # Create secondary ingress controllers
    - kubectl apply -f src/tests/ofc25/nginx-ingress-controller-ip.yaml

    # Deploy TFS for IP
    - source src/tests/ofc25/deploy_specs_ip.sh
    - kubectl apply -f src/tests/${TEST_NAME}/nginx-ingress-controller-ip.yaml

    # Change the name for the database
    - cp manifests/contextservice.yaml manifests/contextservice.yaml.bak
    - |
      sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_ip_context"/}' manifests/contextservice.yaml

    - echo "Sleeping 60"
    - sleep 60
    
    # - source src/tests/${TEST_NAME}/deploy_specs_ip.sh
    - ./deploy/crdb.sh
    - ./deploy/nats.sh
    - ./deploy/kafka.sh
    # - ./deploy/qdb.sh
    - ./deploy/expose_dashboard.sh
    # - ./deploy/expose_dashboard.sh
    - ./deploy/tfs.sh
    - ./deploy/show.sh
    # - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_ip.sh

    - mv manifests/contextservice.yaml.bak manifests/contextservice.yaml
    - mv tfs_runtime_env_vars.sh tfs_runtime_env_vars_ip.sh
    - ./src/tests/${TEST_NAME}/subscription_ws_ip.sh

    #- ./src/tests/${TEST_NAME}/subscription_ws_ip.sh


    # Deploy E2E TeraFlowSDN
    # ===== Deploy End-to-End TeraFlowSDN ===============================================
    - source src/tests/${TEST_NAME}/deploy_specs_e2e.sh



    # Delete secondary ingress controllers
    - kubectl delete -f src/tests/ofc25/nginx-ingress-controller-e2e.yaml --ignore-not-found

    # Create secondary ingress controllers
    - kubectl apply -f src/tests/ofc25/nginx-ingress-controller-e2e.yaml
    - kubectl apply -f src/tests/${TEST_NAME}/nginx-ingress-controller-e2e.yaml

    # Change the name for the database
    - cp manifests/contextservice.yaml manifests/contextservice.yaml.bak
    - |
      sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_e2e_context"/}' manifests/contextservice.yaml

    # - sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_e2e_context"/}' manifests/contextservice.yaml
    
    - ./deploy/crdb.sh
    - ./deploy/nats.sh
    - ./deploy/kafka.sh
    # - ./deploy/qdb.sh
    - ./deploy/expose_dashboard.sh
    # - ./deploy/expose_dashboard.sh
    - ./deploy/tfs.sh
    - ./deploy/show.sh
    # - ./src/tests/${TEST_NAME}/subscription_ws_e2e.sh
    # - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_e2e.sh

    - mv manifests/contextservice.yaml.bak manifests/contextservice.yaml

    #Configure Subscription WS
    - ./src/tests/ofc25/subscription_ws_e2e.sh

    - mv tfs_runtime_env_vars.sh tfs_runtime_env_vars_e2e.sh
    #- ./src/tests/${TEST_NAME}/subscription_ws_e2e.sh


    # Configure Subscription WS
    - ./src/tests/${TEST_NAME}/subscription_ws_e2e.sh

    - echo "Por aqui"
    - sleep 600

    # Run end-to-end tests
    #- if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi
    - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi
    #- >
    #  docker run -t --name ${TEST_NAME} --network=host 
    #  docker run -t --rm --name ${TEST_NAME} --network=host 
    #  --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh"
    #  --volume "$PWD/src/tests/${TEST_NAME}:/opt/results"
    #  $CI_REGISTRY_IMAGE/${TEST_NAME}:latest

  after_script:
    # Dump TeraFlowSDN component logs
    - echo "After"
    
    - source ./src/tests/${TEST_NAME}/deploy_specs_ip.sh
    - kubectl --namespace tfs-ip logs deployment/contextservice -c server
    - kubectl --namespace tfs-ip logs deployment/deviceservice -c server
    - kubectl --namespace tfs-ip logs deployment/pathcompservice -c frontend
    - kubectl --namespace tfs-ip logs deployment/serviceservice -c server
    - kubectl --namespace tfs-ip logs deployment/nbiservice -c server
    - kubectl --namespace tfs-ip logs deployment/vnt-managerservice -c server

    - source ./src/tests/${TEST_NAME}/deploy_specs_opt.sh
    - kubectl --namespace tfs-opt logs deployment/contextservice -c server
    - kubectl --namespace tfs-opt logs deployment/deviceservice -c server
    - kubectl --namespace tfs-opt logs deployment/pathcompservice -c frontend
    - kubectl --namespace tfs-opt logs deployment/serviceservice -c server
    - kubectl --namespace tfs-opt logs deployment/nbiservice -c server

    - source ./src/tests/${TEST_NAME}/deploy_specs_e2e.sh
    - kubectl --namespace tfs-e2e logs deployment/contextservice -c server
    - kubectl --namespace tfs-e2e logs deployment/deviceservice -c server
    - kubectl --namespace tfs-e2e logs deployment/pathcompservice -c frontend
    - kubectl --namespace tfs-e2e logs deployment/serviceservice -c server
    - kubectl --namespace tfs-e2e logs deployment/nbiservice -c server
    - kubectl --namespace tfs-e2e logs deployment/e2e-orchestratorservice -c server


    - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi

    # Dump container status and logs
    # Dump Optical Device Node Agents container status and logs
    - docker ps -a
    - docker logs na-t1
    - docker logs na-t2
    - docker logs na-r1
    - docker logs na-r2

    # Clean old docker images
    - docker image prune --force

    # Dump TeraFlowSDN component logs
    - source src/tests/${TEST_NAME}/deploy_specs_opt.sh
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server

    - source src/tests/${TEST_NAME}/deploy_specs_ip.sh
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/vnt-managerservice -c server

    - helm3 uninstall --namespace nats-e2e nats-e2e 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats-ip nats-ip 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats-opt nats-opt 2>/dev/null  || echo "Namespace not found"
    - helm3 uninstall --namespace nats nats 2>/dev/null  || echo "Namespace not found"
    - kubectl delete namespaces tfs tfs-ip tfs-opt tfs-e2e qdb qdb-e2e qdb-opt qdb-ip --ignore-not-found
    - kubectl delete namespaces nats nats-ip nats-opt nats-e2e --ignore-not-found
    - echo "Adios"
    - sleep 600
    - source src/tests/${TEST_NAME}/deploy_specs_e2e.sh
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server
    - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/e2e-orchestratorservice -c server

    # Clean up
    - docker ps --all --quiet | xargs --no-run-if-empty docker stop
    - docker container prune --force
    - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force
    - docker network prune --force
    - docker volume prune --all --force
    - docker image prune --force

    - kubectl delete namespaces tfs-ip tfs-opt tfs-e2e --ignore-not-found
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-opt.yaml --ignore-not-found
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-ip.yaml --ignore-not-found
    - kubectl delete -f src/tests/${TEST_NAME}/nginx-ingress-controller-e2e.yaml --ignore-not-found
    - helm3 uninstall --namespace nats-e2e nats-e2e || echo "Namespace not found"
    - helm3 uninstall --namespace nats-ip nats-ip || echo "Namespace not found"
    - helm3 uninstall --namespace nats-opt nats-opt || echo "Namespace not found"
    - kubectl delete namespaces nats-ip nats-opt nats-e2e --ignore-not-found
    - kubectl delete namespaces crdb qdb-e2e qdb-opt qdb-ip --ignore-not-found

  #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
  rules: