diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a85988f2716d6524e18eda557bab572f087f56b4..6c95ab8abf405ef5333927de1a949a2a89318a1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,27 +2,36 @@ stages: - lint - dev_deploy - dev_workflows + - dev_cleanup - dev_merge - get_images - update_values - package_publish - release + - nightly_deploy + - nightly_workflows + - nightly_result default: image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16 variables: + HELM_RELEASE_NAME: "test" + HELM_NAMESPACE: "test" + REMOTE_DEPLOY_DIR: "/home/ubuntu/etsi_installer" GIT_DEPTH: 0 - TMF_API_URL: "https://labs.etsi.org/rep/api/v4/projects/470/registry/repositories/3021/tags?page=1&per_page=500" - SONATA_URL: "https://labs.etsi.org/rep/api/v4/projects/479/registry/repositories/3031/tags?page=1&per_page=500" - OSS_CLIENT_URL: "https://labs.etsi.org/rep/api/v4/projects/474/registry/repositories/3023/tags?page=1&per_page=500" - PKG_MANAGER_URL: "https://labs.etsi.org/rep/api/v4/projects/478/registry/repositories/3033/tags?page=1&per_page=500" - PORTAL_CART_URL: "https://labs.etsi.org/rep/api/v4/projects/465/registry/repositories/3041/tags?page=1&per_page=500" - TELEMETRY_CLIENT_URL: "https://labs.etsi.org/rep/api/v4/projects/475/registry/repositories/3028/tags?page=1&per_page=500" - PEERING_API_URL: "https://labs.etsi.org/rep/api/v4/projects/468/registry/repositories/3014/tags?page=1&per_page=500" - REGISTRY_API_URL: "https://labs.etsi.org/rep/api/v4/projects/469/registry/repositories/3018/tags?page=1&per_page=500" - FABRIC_API_URL: "https://labs.etsi.org/rep/api/v4/projects/467/registry/repositories/3011/tags?page=1&per_page=500" - PORTAL_URL: "https://labs.etsi.org/rep/api/v4/projects/466/registry/repositories/3038/tags?page=1&per_page=500" + NIGHTLY_IMAGES: | + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/fabric/fabric-api-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/peering/peering-api-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/registry/registry-api-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/tmf/tmf-api-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/oss/oss-client-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/telemetry/telemetry-api-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/pkg-manager + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/service.monitor/lcm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/sonata/sonata-core-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/cart/hypo-portal-jvm + labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/web/hypo-portal lint: stage: lint @@ -36,6 +45,7 @@ lint: - helm lint hypo except: - main + - develop - gitops-cd - tags @@ -45,12 +55,6 @@ dev_deploy_job: image: name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9 pull_policy: [always, if-not-present] - - variables: - HELM_RELEASE_NAME: "test" - HELM_NAMESPACE: "test" - REMOTE_DEPLOY_DIR: "/home/ubuntu/etsi_installer" - before_script: - mkdir -p /tmp/bin ~/.ssh - chmod 700 ~/.ssh @@ -111,8 +115,10 @@ dev_deploy_job: set -eu sops --decrypt \ + --input-type yaml \ + --output-type yaml \ --output charts/hypo/values-installer.yaml \ - "$HELM_VALUES_ENCRYPTED" + "$HELM_VALUES_FILE" chmod 600 charts/hypo/values-installer.yaml @@ -155,7 +161,6 @@ dev_deploy_job: echo 'Helm deployment completed successfully.' " - rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -166,15 +171,86 @@ dev_workflows_job: image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17 before_script: # clone the integration tests repo - - git clone --depth 1 "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO" + - git clone --depth 1 --branch develop "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO" - cd integration.tests script: - - mvn -B -q test "-Dkarate.env=$KARATE_ENV" "-Dkarate.options=--tags @smoke-e2e-workflow-private" + - mvn -B -q test "-Dkarate.env=$KARATE_ENV_INSTALLER" "-Dkarate.options=--tags @smoke-e2e-workflow-private" dependencies: - dev_deploy_job rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' +dev_cleanup_job: + stage: dev_cleanup + image: + name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9 + pull_policy: [always, if-not-present] + before_script: + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + + # Install SSH client + - | + if command -v apk >/dev/null 2>&1; then + apk add --no-cache openssh-client + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends openssh-client + rm -rf /var/lib/apt/lists/* + else + echo "Unsupported container base image." + exit 1 + fi + + # Configure SSH private key + - chmod 600 "$ETSI_SSH_KEY" + + # Add deployment server host key + - ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + + script: + - | + ssh \ + -i "$ETSI_SSH_KEY" \ + -o IdentitiesOnly=yes \ + "$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" " + set -e + + echo 'Uninstalling Helm release...' + + helm uninstall '$HELM_RELEASE_NAME' \ + --namespace '$HELM_NAMESPACE' \ + --wait \ + --timeout 10m \ + || echo 'Helm release does not exist or was already removed.' + + echo 'Deleting PVCs from namespace...' + + kubectl delete pvc \ + --all \ + --namespace '$HELM_NAMESPACE' \ + --ignore-not-found=true \ + --wait=true \ + --timeout=10m + + echo 'Deleting namespace...' + + kubectl delete namespace '$HELM_NAMESPACE' \ + --ignore-not-found=true \ + --wait=true \ + --timeout=10m + + echo 'Removing copied Helm chart...' + + rm -rf '$REMOTE_DEPLOY_DIR' + + echo 'Cleanup completed successfully.' + " + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + dev_merge: stage: dev_merge @@ -195,7 +271,6 @@ dev_merge: - git push origin gitops-cd rules: - if: '$CI_COMMIT_BRANCH == "main"' - #- if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' get_images: @@ -379,3 +454,302 @@ release_job: variables: - '$CI_COMMIT_MESSAGE =~ /^release:\s/' + +nightly_deploy_job: + stage: nightly_deploy + image: + name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9 + pull_policy: [always, if-not-present] + before_script: + - mkdir -p /tmp/bin ~/.ssh + - chmod 700 ~/.ssh + - export PATH="/tmp/bin:$PATH" + + # Install SSH client and required utilities + - | + if command -v apk >/dev/null 2>&1; then + apk add --no-cache openssh-client curl tar + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends \ + openssh-client curl tar + rm -rf /var/lib/apt/lists/* + else + echo "Unsupported container base image." + exit 1 + fi + + # Install age + - curl -L -o /tmp/age.tar.gz https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz + - tar -xzf /tmp/age.tar.gz -C /tmp + - mv /tmp/age/age /tmp/age/age-keygen /tmp/bin/ + + # Install sops + - curl -L -o /tmp/bin/sops https://github.com/getsops/sops/releases/download/v3.10.2/sops-v3.10.2.linux.amd64 + - chmod +x /tmp/bin/sops + + - sops --version + - age --version + + # Configure the SSH private key + - chmod 600 "$ETSI_SSH_KEY" + + # SOPS key + - echo "$SOPS_AGE_KEY_PRIVATE" > ~/.sops-age-key.txt + - chmod 600 ~/.sops-age-key.txt + - export SOPS_AGE_KEY_FILE=~/.sops-age-key.txt + + # Add the deployment server host key + - ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + + # Verify SSH connection + - | + ssh \ + -i "$ETSI_SSH_KEY" \ + -o IdentitiesOnly=yes \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + "$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" \ + 'echo "SSH connection successful on $(hostname)"' + + script: + + # Decrypt the installer values directly inside the chart + - | + set -eu + + sops --decrypt \ + --input-type yaml \ + --output-type yaml \ + --output charts/hypo/values-nightly.yaml \ + "$HELM_NIGHTLY_VALUES_FILE" + + chmod 600 charts/hypo/values-nightly.yaml + + # Remove and recreate the temporary remote directory + - | + ssh \ + -i "$ETSI_SSH_KEY" \ + -o IdentitiesOnly=yes \ + "$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" " + rm -rf '$REMOTE_DEPLOY_DIR' + mkdir -p '$REMOTE_DEPLOY_DIR' + " + - | + scp \ + -i "$ETSI_SSH_KEY" \ + -o IdentitiesOnly=yes \ + -r \ + charts/hypo/. \ + "$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP:$REMOTE_DEPLOY_DIR/" + + # Build dependencies and deploy + - | + ssh \ + -i "$ETSI_SSH_KEY" \ + -o IdentitiesOnly=yes \ + "$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" " + + set -e + + cd '$REMOTE_DEPLOY_DIR' + + helm dependency build . + + helm upgrade --install nightly . \ + --namespace nightly \ + --create-namespace \ + --values values-nightly.yaml \ + --wait \ + --timeout 10m + + echo 'Helm deployment completed successfully.' + " + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: on_success + - when: never + +nightly_workflows_job: + stage: nightly_workflows + image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17 + before_script: + # clone the integration tests repo + - git clone --depth 1 --branch develop "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO" + - cd integration.tests + script: + - mvn -B -q test "-Dkarate.env=$KARATE_ENV_INSTALLER" "-Dkarate.options=--tags @smoke-e2e-workflow-private" + needs: + - job: nightly_deploy_job + artifacts: false + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: on_success + - when: never + + + +nightly_promote_images_job: + stage: nightly_result + tags: + - hypo + image: + name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3 + needs: + - job: nightly_workflows_job + artifacts: false + before_script: + - docker info >/dev/null + - | + echo -n "$CI_REGISTRY_PASSWORD" | + docker login \ + --username "$CI_REGISTRY_USER" \ + --password-stdin \ + "$CI_REGISTRY" + script: + - | + set -eu + + DATE_TAG="nightly-$(date +%Y-%m-%d)" + + for IMAGE in $NIGHTLY_IMAGES; do + DEV_IMAGE="${IMAGE}:dev-latest" + LATEST_IMAGE="${IMAGE}:latest" + DATED_IMAGE="${IMAGE}:${DATE_TAG}" + + docker pull "$DEV_IMAGE" + + docker tag "$DEV_IMAGE" "$LATEST_IMAGE" + docker tag "$DEV_IMAGE" "$DATED_IMAGE" + + docker push "$LATEST_IMAGE" + docker push "$DATED_IMAGE" + + docker image rm -f "$DEV_IMAGE" "$LATEST_IMAGE" "$DATED_IMAGE" >/dev/null 2>&1 || true + done + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: on_success + - when: never + + +nightly_compare_failed_images_job: + stage: nightly_result + tags: + - hypo + image: + name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3 + needs: + - job: nightly_workflows_job + artifacts: false + optional: true + before_script: + - docker info >/dev/null + - | + echo -n "$CI_REGISTRY_PASSWORD" | + docker login \ + --username "$CI_REGISTRY_USER" \ + --password-stdin \ + "$CI_REGISTRY" + + script: + - | + set -eu + + REPORT_FILE="different_images.txt" + FAILED_DATE_TAG="failed-nightly-$(date +%Y-%m-%d)" + + : > "$REPORT_FILE" + + echo "Nightly workflows failed." + echo "Comparing dev-latest images with latest..." + echo + + for IMAGE in $NIGHTLY_IMAGES; do + DEV_IMAGE="${IMAGE}:dev-latest" + LATEST_IMAGE="${IMAGE}:latest" + FAILED_IMAGE="${IMAGE}:${FAILED_DATE_TAG}" + + echo "==============================================" + echo "Checking: $IMAGE" + + if ! docker pull "$DEV_IMAGE"; then + echo "$IMAGE - dev-latest could not be pulled" \ + >> "$REPORT_FILE" + + echo "Skipping rollback because dev-latest is unavailable." + continue + fi + + if ! docker pull "$LATEST_IMAGE"; then + echo "$IMAGE - latest could not be pulled" \ + >> "$REPORT_FILE" + + echo "Skipping rollback because latest is unavailable." + + docker image rm -f "$DEV_IMAGE" \ + >/dev/null 2>&1 || true + + continue + fi + + DEV_IMAGE_ID=$( + docker image inspect \ + --format '{{.Id}}' \ + "$DEV_IMAGE" + ) + + LATEST_IMAGE_ID=$( + docker image inspect \ + --format '{{.Id}}' \ + "$LATEST_IMAGE" + ) + + if [ "$DEV_IMAGE_ID" != "$LATEST_IMAGE_ID" ]; then + echo "Result: DIFFERENT" + echo "$IMAGE" >> "$REPORT_FILE" + + # Preserve the failed dev-latest image + docker tag "$DEV_IMAGE" "$FAILED_IMAGE" + docker push "$FAILED_IMAGE" + + # Replace dev-latest with the last successful latest image + docker tag "$LATEST_IMAGE" "$DEV_IMAGE" + docker push "$DEV_IMAGE" + + echo "Saved failed image as:" + echo " $FAILED_IMAGE" + + echo "Restored dev-latest from:" + echo " $LATEST_IMAGE" + else + echo "Result: SAME" + fi + + docker image rm -f \ + "$DEV_IMAGE" \ + "$LATEST_IMAGE" \ + "$FAILED_IMAGE" \ + >/dev/null 2>&1 || true + done + + echo + echo "Images requiring investigation:" + echo "==============================================" + + if [ -s "$REPORT_FILE" ]; then + cat "$REPORT_FILE" + else + echo "No differences were found." + fi + + artifacts: + when: always + expire_in: 30 days + paths: + - different_images.txt + + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: on_failure