Merge branch 'test-ci' into 'main' (74772f50) · Commits · OSL / MDG HypO / code / org.etsi.osl.hypo.ops / org.etsi.osl.hypo.ops.installer

.gitlab-ci.yml

+395 −21

Original line number	Diff line number	Diff line
		@@ -2,27 +2,36 @@ stages:
		- lint
		- dev_deploy
		- dev_workflows
		- dev_cleanup
		- dev_merge
		- get_images
		- update_values
		- package_publish
		- release
		- nightly_deploy
		- nightly_workflows
		- nightly_result

		default:
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16

		variables:
		HELM_RELEASE_NAME: "test"
		HELM_NAMESPACE: "test"
		REMOTE_DEPLOY_DIR: "/home/ubuntu/etsi_installer"
		GIT_DEPTH: 0
		TMF_API_URL: "https://labs.etsi.org/rep/api/v4/projects/470/registry/repositories/3021/tags?page=1&per_page=500"
		SONATA_URL: "https://labs.etsi.org/rep/api/v4/projects/479/registry/repositories/3031/tags?page=1&per_page=500"
		OSS_CLIENT_URL: "https://labs.etsi.org/rep/api/v4/projects/474/registry/repositories/3023/tags?page=1&per_page=500"
		PKG_MANAGER_URL: "https://labs.etsi.org/rep/api/v4/projects/478/registry/repositories/3033/tags?page=1&per_page=500"
		PORTAL_CART_URL: "https://labs.etsi.org/rep/api/v4/projects/465/registry/repositories/3041/tags?page=1&per_page=500"
		TELEMETRY_CLIENT_URL: "https://labs.etsi.org/rep/api/v4/projects/475/registry/repositories/3028/tags?page=1&per_page=500"
		PEERING_API_URL: "https://labs.etsi.org/rep/api/v4/projects/468/registry/repositories/3014/tags?page=1&per_page=500"
		REGISTRY_API_URL: "https://labs.etsi.org/rep/api/v4/projects/469/registry/repositories/3018/tags?page=1&per_page=500"
		FABRIC_API_URL: "https://labs.etsi.org/rep/api/v4/projects/467/registry/repositories/3011/tags?page=1&per_page=500"
		PORTAL_URL: "https://labs.etsi.org/rep/api/v4/projects/466/registry/repositories/3038/tags?page=1&per_page=500"
		NIGHTLY_IMAGES: \|
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/fabric/fabric-api-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/peering/peering-api-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/registry/registry-api-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.api/tmf/tmf-api-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/oss/oss-client-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/telemetry/telemetry-api-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/pkg-manager
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/service.monitor/lcm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/sonata/sonata-core-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/cart/hypo-portal-jvm
		labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/web/hypo-portal

		lint:
		stage: lint
		@@ -36,6 +45,7 @@ lint:
		- helm lint hypo
		except:
		- main
		- develop
		- gitops-cd
		- tags

		@@ -45,12 +55,6 @@ dev_deploy_job:
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9
		pull_policy: [always, if-not-present]

		variables:
		HELM_RELEASE_NAME: "test"
		HELM_NAMESPACE: "test"
		REMOTE_DEPLOY_DIR: "/home/ubuntu/etsi_installer"

		before_script:
		- mkdir -p /tmp/bin ~/.ssh
		- chmod 700 ~/.ssh
		@@ -111,8 +115,10 @@ dev_deploy_job:
		set -eu

		sops --decrypt \
		--input-type yaml \
		--output-type yaml \
		--output charts/hypo/values-installer.yaml \
		"$HELM_VALUES_ENCRYPTED"
		"$HELM_VALUES_FILE"

		chmod 600 charts/hypo/values-installer.yaml

		@@ -155,7 +161,6 @@ dev_deploy_job:

		echo 'Helm deployment completed successfully.'
		"

		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

		@@ -166,15 +171,86 @@ dev_workflows_job:
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17
		before_script:
		# clone the integration tests repo
		- git clone --depth 1 "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO"
		- git clone --depth 1 --branch develop "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO"
		- cd integration.tests
		script:
		- mvn -B -q test "-Dkarate.env=$KARATE_ENV" "-Dkarate.options=--tags @smoke-e2e-workflow-private"
		- mvn -B -q test "-Dkarate.env=$KARATE_ENV_INSTALLER" "-Dkarate.options=--tags @smoke-e2e-workflow-private"
		dependencies:
		- dev_deploy_job
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

		dev_cleanup_job:
		stage: dev_cleanup
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9
		pull_policy: [always, if-not-present]
		before_script:
		- mkdir -p ~/.ssh
		- chmod 700 ~/.ssh

		# Install SSH client
		- \|
		if command -v apk >/dev/null 2>&1; then
		apk add --no-cache openssh-client
		elif command -v apt-get >/dev/null 2>&1; then
		apt-get update
		apt-get install -y --no-install-recommends openssh-client
		rm -rf /var/lib/apt/lists/*
		else
		echo "Unsupported container base image."
		exit 1
		fi

		# Configure SSH private key
		- chmod 600 "$ETSI_SSH_KEY"

		# Add deployment server host key
		- ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts
		- chmod 644 ~/.ssh/known_hosts

		script:
		- \|
		ssh \
		-i "$ETSI_SSH_KEY" \
		-o IdentitiesOnly=yes \
		"$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" "
		set -e

		echo 'Uninstalling Helm release...'

		helm uninstall '$HELM_RELEASE_NAME' \
		--namespace '$HELM_NAMESPACE' \
		--wait \
		--timeout 10m \
		\|\| echo 'Helm release does not exist or was already removed.'

		echo 'Deleting PVCs from namespace...'

		kubectl delete pvc \
		--all \
		--namespace '$HELM_NAMESPACE' \
		--ignore-not-found=true \
		--wait=true \
		--timeout=10m

		echo 'Deleting namespace...'

		kubectl delete namespace '$HELM_NAMESPACE' \
		--ignore-not-found=true \
		--wait=true \
		--timeout=10m

		echo 'Removing copied Helm chart...'

		rm -rf '$REMOTE_DEPLOY_DIR'

		echo 'Cleanup completed successfully.'
		"
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
		when: always


		dev_merge:
		stage: dev_merge
		@@ -195,7 +271,6 @@ dev_merge:
		- git push origin gitops-cd
		rules:
		- if: '$CI_COMMIT_BRANCH == "main"'
		#- if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'


		get_images:
		@@ -379,3 +454,302 @@ release_job:
		variables:
		- '$CI_COMMIT_MESSAGE =~ /^release:\s/'


		nightly_deploy_job:
		stage: nightly_deploy
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9
		pull_policy: [always, if-not-present]
		before_script:
		- mkdir -p /tmp/bin ~/.ssh
		- chmod 700 ~/.ssh
		- export PATH="/tmp/bin:$PATH"

		# Install SSH client and required utilities
		- \|
		if command -v apk >/dev/null 2>&1; then
		apk add --no-cache openssh-client curl tar
		elif command -v apt-get >/dev/null 2>&1; then
		apt-get update
		apt-get install -y --no-install-recommends \
		openssh-client curl tar
		rm -rf /var/lib/apt/lists/*
		else
		echo "Unsupported container base image."
		exit 1
		fi

		# Install age
		- curl -L -o /tmp/age.tar.gz https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz
		- tar -xzf /tmp/age.tar.gz -C /tmp
		- mv /tmp/age/age /tmp/age/age-keygen /tmp/bin/

		# Install sops
		- curl -L -o /tmp/bin/sops https://github.com/getsops/sops/releases/download/v3.10.2/sops-v3.10.2.linux.amd64
		- chmod +x /tmp/bin/sops

		- sops --version
		- age --version

		# Configure the SSH private key
		- chmod 600 "$ETSI_SSH_KEY"

		# SOPS key
		- echo "$SOPS_AGE_KEY_PRIVATE" > ~/.sops-age-key.txt
		- chmod 600 ~/.sops-age-key.txt
		- export SOPS_AGE_KEY_FILE=~/.sops-age-key.txt

		# Add the deployment server host key
		- ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts
		- chmod 644 ~/.ssh/known_hosts

		# Verify SSH connection
		- \|
		ssh \
		-i "$ETSI_SSH_KEY" \
		-o IdentitiesOnly=yes \
		-o BatchMode=yes \
		-o ConnectTimeout=10 \
		"$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" \
		'echo "SSH connection successful on $(hostname)"'

		script:

		# Decrypt the installer values directly inside the chart
		- \|
		set -eu

		sops --decrypt \
		--input-type yaml \
		--output-type yaml \
		--output charts/hypo/values-nightly.yaml \
		"$HELM_NIGHTLY_VALUES_FILE"

		chmod 600 charts/hypo/values-nightly.yaml

		# Remove and recreate the temporary remote directory
		- \|
		ssh \
		-i "$ETSI_SSH_KEY" \
		-o IdentitiesOnly=yes \
		"$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" "
		rm -rf '$REMOTE_DEPLOY_DIR'
		mkdir -p '$REMOTE_DEPLOY_DIR'
		"
		- \|
		scp \
		-i "$ETSI_SSH_KEY" \
		-o IdentitiesOnly=yes \
		-r \
		charts/hypo/. \
		"$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP:$REMOTE_DEPLOY_DIR/"

		# Build dependencies and deploy
		- \|
		ssh \
		-i "$ETSI_SSH_KEY" \
		-o IdentitiesOnly=yes \
		"$DEPLOYMENT_SERVER_USER@$DEPLOYMENT_SERVER_IP" "

		set -e

		cd '$REMOTE_DEPLOY_DIR'

		helm dependency build .

		helm upgrade --install nightly . \
		--namespace nightly \
		--create-namespace \
		--values values-nightly.yaml \
		--wait \
		--timeout 10m

		echo 'Helm deployment completed successfully.'
		"
		rules:
		- if: '$CI_PIPELINE_SOURCE == "schedule"'
		when: on_success
		- when: never

		nightly_workflows_job:
		stage: nightly_workflows
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17
		before_script:
		# clone the integration tests repo
		- git clone --depth 1 --branch develop "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO"
		- cd integration.tests
		script:
		- mvn -B -q test "-Dkarate.env=$KARATE_ENV_INSTALLER" "-Dkarate.options=--tags @smoke-e2e-workflow-private"
		needs:
		- job: nightly_deploy_job
		artifacts: false
		rules:
		- if: '$CI_PIPELINE_SOURCE == "schedule"'
		when: on_success
		- when: never



		nightly_promote_images_job:
		stage: nightly_result
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3
		needs:
		- job: nightly_workflows_job
		artifacts: false
		before_script:
		- docker info >/dev/null
		- \|
		echo -n "$CI_REGISTRY_PASSWORD" \|
		docker login \
		--username "$CI_REGISTRY_USER" \
		--password-stdin \
		"$CI_REGISTRY"
		script:
		- \|
		set -eu

		DATE_TAG="nightly-$(date +%Y-%m-%d)"

		for IMAGE in $NIGHTLY_IMAGES; do
		DEV_IMAGE="${IMAGE}:dev-latest"
		LATEST_IMAGE="${IMAGE}:latest"
		DATED_IMAGE="${IMAGE}:${DATE_TAG}"

		docker pull "$DEV_IMAGE"

		docker tag "$DEV_IMAGE" "$LATEST_IMAGE"
		docker tag "$DEV_IMAGE" "$DATED_IMAGE"

		docker push "$LATEST_IMAGE"
		docker push "$DATED_IMAGE"

		docker image rm -f "$DEV_IMAGE" "$LATEST_IMAGE" "$DATED_IMAGE" >/dev/null 2>&1 \|\| true
		done
		rules:
		- if: '$CI_PIPELINE_SOURCE == "schedule"'
		when: on_success
		- when: never


		nightly_compare_failed_images_job:
		stage: nightly_result
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3
		needs:
		- job: nightly_workflows_job
		artifacts: false
		optional: true
		before_script:
		- docker info >/dev/null
		- \|
		echo -n "$CI_REGISTRY_PASSWORD" \|
		docker login \
		--username "$CI_REGISTRY_USER" \
		--password-stdin \
		"$CI_REGISTRY"

		script:
		- \|
		set -eu

		REPORT_FILE="different_images.txt"
		FAILED_DATE_TAG="failed-nightly-$(date +%Y-%m-%d)"

		: > "$REPORT_FILE"

		echo "Nightly workflows failed."
		echo "Comparing dev-latest images with latest..."
		echo

		for IMAGE in $NIGHTLY_IMAGES; do
		DEV_IMAGE="${IMAGE}:dev-latest"
		LATEST_IMAGE="${IMAGE}:latest"
		FAILED_IMAGE="${IMAGE}:${FAILED_DATE_TAG}"

		echo "=============================================="
		echo "Checking: $IMAGE"

		if ! docker pull "$DEV_IMAGE"; then
		echo "$IMAGE - dev-latest could not be pulled" \
		>> "$REPORT_FILE"

		echo "Skipping rollback because dev-latest is unavailable."
		continue
		fi

		if ! docker pull "$LATEST_IMAGE"; then
		echo "$IMAGE - latest could not be pulled" \
		>> "$REPORT_FILE"

		echo "Skipping rollback because latest is unavailable."

		docker image rm -f "$DEV_IMAGE" \
		>/dev/null 2>&1 \|\| true

		continue
		fi

		DEV_IMAGE_ID=$(
		docker image inspect \
		--format '{{.Id}}' \
		"$DEV_IMAGE"
		)

		LATEST_IMAGE_ID=$(
		docker image inspect \
		--format '{{.Id}}' \
		"$LATEST_IMAGE"
		)

		if [ "$DEV_IMAGE_ID" != "$LATEST_IMAGE_ID" ]; then
		echo "Result: DIFFERENT"
		echo "$IMAGE" >> "$REPORT_FILE"

		# Preserve the failed dev-latest image
		docker tag "$DEV_IMAGE" "$FAILED_IMAGE"
		docker push "$FAILED_IMAGE"

		# Replace dev-latest with the last successful latest image
		docker tag "$LATEST_IMAGE" "$DEV_IMAGE"
		docker push "$DEV_IMAGE"

		echo "Saved failed image as:"
		echo " $FAILED_IMAGE"

		echo "Restored dev-latest from:"
		echo " $LATEST_IMAGE"
		else
		echo "Result: SAME"
		fi

		docker image rm -f \
		"$DEV_IMAGE" \
		"$LATEST_IMAGE" \
		"$FAILED_IMAGE" \
		>/dev/null 2>&1 \|\| true
		done

		echo
		echo "Images requiring investigation:"
		echo "=============================================="

		if [ -s "$REPORT_FILE" ]; then
		cat "$REPORT_FILE"
		else
		echo "No differences were found."
		fi

		artifacts:
		when: always
		expire_in: 30 days
		paths:
		- different_images.txt

		rules:
		- if: '$CI_PIPELINE_SOURCE == "schedule"'
		when: on_failure