Merge branch 'ci/hub-upgrade-pipeline' into 'main' (b9928e65) · Commits · OSL / MDG HypO / code / addons / ocm-scheduler

.gitignore

0 → 100644

+3 −0

Changes for .gitignore: 3 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		todo.md
		docs/concepts.md
		etsi-migration.txt

.gitlab-ci.yml

0 → 100644

+316 −0

Changes for .gitlab-ci.yml: 316 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		stages:
		- lint
		- test
		- analyze
		- dev_image_build
		- hub_deploy
		- dev_workflows
		- hub_verify
		- promote
		- release
		- image_build

		default:
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/golang:latest

		variables:
		DOCKER_HOST: unix:///var/run/docker.sock
		DOCKER_DRIVER: overlay2
		IMAGE_NAME: "hypo-ocm-scheduler"
		OPERATOR_DIR: "src/ocm_scheduler_operator"
		CHART_DIR: "chart/ocm-scheduler-chart"
		SONAR_GO_COVERAGE: "${CI_PROJECT_DIR}/sonarQube"
		INTEGRATION_BRANCH: "develop"

		.go-cache:
		variables:
		GOPATH: $CI_PROJECT_DIR/.go
		cache:
		paths:
		- .go/pkg/mod/

		# The OCM hub and the scheduler chart (helm release ocm-scheduler in hypo-scheduler-system) are installed
		# by hand before the first run. MR pipelines only swap the image of the scheduler Deployment and put the
		# previous image back if the MR fails: chart, CRD and RBAC changes are applied to the hub by hand (chart/README.md).
		.hub:
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/alpine/k8s:1.31.1
		entrypoint: [""]
		# one pipeline at a time on the OCM hub
		resource_group: ocm-hub
		variables:
		NS: "hypo-scheduler-system"
		DEPLOY: "ocm-scheduler-controller-manager"
		CONTAINER: "manager"
		NEW_IMAGE: "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA"
		before_script:
		- chmod 600 "$OCM_HUB_KUBECONFIG"
		- export KUBECONFIG="$OCM_HUB_KUBECONFIG"
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'


		lint:
		stage: lint
		tags:
		- hypo
		extends: .go-cache
		before_script:
		- mkdir -p ${SONAR_GO_COVERAGE}
		- cd $OPERATOR_DIR
		script:
		- go mod download
		- make golangci-lint
		- bin/golangci-lint -c .golangci.yaml run --out-format=colored-line-number,checkstyle:${SONAR_GO_COVERAGE}/golanci-report.xml
		artifacts:
		when: always
		paths:
		- ${SONAR_GO_COVERAGE}
		except:
		- main
		- tags
		- develop

		chart_lint:
		stage: lint
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/alpine/helm:3.17.2
		entrypoint: [""]
		script:
		- helm lint $CHART_DIR
		- helm template test $CHART_DIR > /dev/null
		except:
		- main
		- tags
		- develop

		tests_job:
		stage: test
		tags:
		- hypo
		extends: .go-cache
		before_script:
		- mkdir -p ${SONAR_GO_COVERAGE}
		- cd $OPERATOR_DIR
		script:
		- go mod download
		- make test
		- cp cover.out ${SONAR_GO_COVERAGE}/coverage.out
		artifacts:
		paths:
		- ${SONAR_GO_COVERAGE}
		except:
		- main
		- tags
		- develop

		sonar-analysis:
		stage: analyze
		tags:
		- hypo
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/sonarsource/sonar-scanner-cli:latest
		script:
		- sonar-scanner -Dsonar.projectKey=${CI_PROJECT_TITLE} -Dsonar.sources=${OPERATOR_DIR} -Dsonar.exclusions=*/zz_generated.go -Dsonar.coverage.exclusions=*/_test.go -Dsonar.go.coverage.reportPaths=${SONAR_GO_COVERAGE}/coverage.out -Dsonar.go.golangci-lint.reportPaths=${SONAR_GO_COVERAGE}/golanci-report.xml
		allow_failure: true
		needs:
		- job: lint
		artifacts: true
		- job: tests_job
		artifacts: true
		artifacts:
		when: always
		paths:
		- ${SONAR_GO_COVERAGE}
		expire_in: 1 day
		except:
		- main
		- tags
		- develop


		dev_image_build_job:
		stage: dev_image_build
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3
		variables:
		BUILT_IMAGE: "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA"
		before_script:
		- docker info > /dev/null
		- echo -n $CI_REGISTRY_PASSWORD \| docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
		script:
		- >
		docker build
		--pull
		--network host
		-t $BUILT_IMAGE
		-f $OPERATOR_DIR/Dockerfile $OPERATOR_DIR
		- docker push $BUILT_IMAGE
		after_script:
		- docker image rm -f "$BUILT_IMAGE" \|\| true

		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'

		# Puts the image of this MR on the scheduler Deployment. The previous image (PREV_IMAGE) is passed on to
		# hub_verify for the revert
		hub_upgrade:
		extends: .hub
		stage: hub_deploy
		needs:
		- job: dev_image_build_job
		script:
		- PREV_IMAGE=$(kubectl get deploy $DEPLOY -n $NS -o jsonpath="{.spec.template.spec.containers[?(@.name=='$CONTAINER')].image}")
		- test -n "$PREV_IMAGE" \|\| { echo "No container $CONTAINER in deploy/$DEPLOY -n $NS"; exit 1; }
		- echo "PREV_IMAGE=$PREV_IMAGE" > hub.env
		- echo "Hub runs $PREV_IMAGE, upgrading to $NEW_IMAGE"
		- kubectl set image deploy/$DEPLOY $CONTAINER=$NEW_IMAGE -n $NS
		- kubectl rollout status deploy/$DEPLOY -n $NS --timeout=5m
		artifacts:
		when: always
		reports:
		dotenv: hub.env


		# Runs the OCM scenario as HypO service orders against the hub: CMS app scheduled by the scheduler of this MR
		dev_workflows_job:
		stage: dev_workflows
		tags:
		- hypo
		image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17
		# one pipeline at a time on the OCM hub
		resource_group: ocm-hub
		before_script:
		- git clone --depth 1 --branch "$INTEGRATION_BRANCH" "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO"
		- cd integration.tests
		script:
		- export KARATE_ENV="${KARATE_ENV:-dev-etsi}"
		- echo "Testing ocm-scheduler image :$CI_COMMIT_SHORT_SHA on Karate env $KARATE_ENV"
		# The result is passed to hub_verify, which keeps or reverts the hub
		- \|
		if mvn -B -q test "-Dkarate.env=$KARATE_ENV" "-Dkarate.options=--tags @smoke-e2e-workflow-ocm"; then
		echo "TEST_RESULT=PASS" > $CI_PROJECT_DIR/result.env
		else
		echo "TEST_RESULT=FAIL" > $CI_PROJECT_DIR/result.env
		exit 1
		fi
		artifacts:
		when: always
		reports:
		dotenv: result.env
		paths:
		- integration.tests/target/karate-reports/
		needs:
		- job: hub_upgrade
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'


		# Keeps the image of this MR on the hub if the tests passed and the scheduler is healthy, otherwise
		# puts PREV_IMAGE back.
		hub_verify:
		extends: .hub
		stage: hub_verify
		needs:
		- job: hub_upgrade
		artifacts: true
		- job: dev_workflows_job
		artifacts: true
		optional: true
		script:
		- \|
		CURRENT=$(kubectl get deploy $DEPLOY -n $NS -o jsonpath="{.spec.template.spec.containers[?(@.name=='$CONTAINER')].image}")
		echo "Test result: ${TEST_RESULT:-none}, hub runs $CURRENT, previous ${PREV_IMAGE:-none}"

		if [ "${TEST_RESULT:-}" = "PASS" ] && [ "$CURRENT" = "$NEW_IMAGE" ] &&
		kubectl rollout status deploy/$DEPLOY -n $NS --timeout=60s; then
		echo "OK: $NEW_IMAGE stays on the hub"
		exit 0
		fi

		if [ -z "${PREV_IMAGE:-}" ] \|\| [ "$CURRENT" = "$PREV_IMAGE" ]; then
		echo "The hub is not on the image of this MR: nothing to revert"
		else
		echo "Reverting the hub to $PREV_IMAGE"
		kubectl set image deploy/$DEPLOY $CONTAINER=$PREV_IMAGE -n $NS
		kubectl rollout status deploy/$DEPLOY -n $NS --timeout=5m
		fi
		exit 1
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'
		when: always

		# The image that passed on the hub becomes :latest (retag only)
		promote_job:
		stage: promote
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3
		before_script:
		- echo -n $CI_REGISTRY_PASSWORD \| docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
		script:
		- echo "Promoting $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA to latest"
		- docker buildx imagetools create -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA
		needs:
		- job: hub_verify
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'


		semantic_versioning_job:
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/node:lts
		stage: release
		tags:
		- hypo
		variables:
		GITLAB_TOKEN: $GROUP_ACCESS_TOKEN
		before_script:
		- git fetch origin main
		- git reset --hard origin/main
		script:
		- npx -y
		-p @semantic-release/commit-analyzer@9.0.2
		-p @semantic-release/git@10.0.1
		-p @semantic-release/gitlab@9.5.0
		-p @semantic-release/release-notes-generator@10.0.3
		-p @semantic-release/exec@6.0.3
		-p @semantic-release/changelog@6.0.2
		-p conventional-changelog-conventionalcommits@5.0.0
		-p semantic-release@19.0.5
		semantic-release
		- RELEASE_VERSION=$(cat .next-release-version 2>/dev/null \|\| echo "no-release")
		- echo "$RELEASE_VERSION" > release_version.txt
		- cat release_version.txt
		artifacts:
		paths:
		- release_version.txt
		only:
		- main


		# Release: the release version is added to :latest, the image that passed on the hub (retag only)
		image_build_job:
		stage: image_build
		tags:
		- hypo
		image:
		name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:29.1.3
		before_script:
		- RELEASE_VERSION=$(cat release_version.txt)
		- echo $RELEASE_VERSION
		- if [ "$RELEASE_VERSION" = "no-release" ]; then exit 0; fi
		- echo -n $CI_REGISTRY_PASSWORD \| docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
		script:
		- docker buildx imagetools create -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$RELEASE_VERSION $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest

		dependencies:
		- semantic_versioning_job
		only:
		- main

.releaserc.json

0 → 100644

+34 −0

Changes for .releaserc.json: 34 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		{
		"branches": [
		"main"
		],
		"plugins": [
		["@semantic-release/commit-analyzer", {
		"preset": "conventionalcommits",
		"presetConfig": {
		"types": [
		{"type": "feat", "section": "Features"},
		{"type": "fix", "section": "Bug Fixes"},
		{"type": "chore", "hidden": true},
		{"type": "docs", "hidden": true},
		{"type": "style", "hidden": true},
		{"type": "refactor", "hidden": true},
		{"type": "perf", "hidden": true},
		{"type": "test", "hidden": true}
		],
		"issueUrlFormat": "https://labs.etsi.org/osl/hypo/code/addons/ocm-scheduler/-/issues/{{id}}"
		}
		}],
		"@semantic-release/release-notes-generator",
		["@semantic-release/gitlab", {
		"gitlabUrl": "https://labs.etsi.org/osl/hypo/code/addons/ocm-scheduler"
		}],
		["@semantic-release/changelog",{
		"changelogFile": "./CHANGELOG.md"
		}],
		["@semantic-release/exec", {
		"verifyReleaseCmd": "echo ${nextRelease.version} > .next-release-version"
		}],
		"@semantic-release/git"
		]
		}

chart/README.md

+36 −8

Changes for chart/README.md: 36 added lines, 8 removed lines.
Original line number	Diff line number	Diff line
		@@ -100,11 +100,11 @@ clusterSet:
		managedClusters:
		- name: managed-1
		location: managed-1
		kubeconfigSecretName: managed1-kubeconfig
		kubeconfigSecretId: <hypo-registry-kubeconfig-id>

		- name: managed-2
		location: managed-2
		kubeconfigSecretName: managed2-kubeconfig
		kubeconfigSecretId: <hypo-registry-kubeconfig-id>
		```

		### Scheduler configuration
		@@ -128,9 +128,9 @@ managedClusters:

		\| Parameter \| Description \|
		\|---\|---\|
		\| `managedClusters[].name` \| Name of the ManagedCluster registered on the OCM Hub. \|
		\| `managedClusters[].name` \| Name of the ManagedCluster registered on the OCM Hub. Also the name of the Kubernetes Secret holding its kubeconfig. \|
		\| `managedClusters[].location` \| Location value assigned to the cluster through the `hypo.filter.location` ClusterClaim. \|
		\| `managedClusters[].kubeconfigSecretName` \| Name of the Kubernetes Secret containing the managed cluster kubeconfig. \|
		\| `managedClusters[].kubeconfigSecretId` \| HypO registry ID of the managed cluster kubeconfig. When deployed from HypO, the package manager fetches it and creates the Secret `<name>` before install. Not used by the chart templates. \|

		The `location` value is used by the scheduler when evaluating location-based scheduling requirements. It should match the location expected by the corresponding workload scheduling configuration.

		@@ -138,16 +138,20 @@ The `location` value is used by the scheduler when evaluating location-based sch

		The Helm setup and cleanup Jobs require access to the managed clusters to create and remove ClusterClaims.

		If we deploy manually the Hlem chart and not from HypO, we need for each managed cluster, to create a Kubernetes Secret in the scheduler namespace containing its kubeconfig.
		When deployed from HypO, the package manager creates these Secrets from `managedClusters[].kubeconfigSecretId` before installing the chart.

		If we deploy manually the Helm chart and not from HypO, we need for each managed cluster, to create a Kubernetes Secret in the scheduler namespace containing its kubeconfig, named after the cluster's `name`.

		This is the case for the OCM hub used by this repository's CI (see [The OCM hub used by the CI](#the-ocm-hub-used-by-the-ci)). The Secrets must stay in place there, since the setup Job reads them again on every `helm upgrade`.

		Example:

		```bash
		kubectl create secret generic managed1-kubeconfig \
		kubectl create secret generic managed-1 \
		--from-file=kubeconfig=/home/ubuntu/managed1.yaml \
		-n hypo-scheduler-system

		kubectl create secret generic managed2-kubeconfig \
		kubectl create secret generic managed-2 \
		--from-file=kubeconfig=/home/ubuntu/managed2.yaml \
		-n hypo-scheduler-system
		```
		@@ -160,7 +164,7 @@ Verify the Secrets:
		kubectl get secrets -n hypo-scheduler-system
		```

		The Secret names must match the `kubeconfigSecretName` values in `values.yaml`.
		The Secret names must match the `managedClusters[].name` values in `values.yaml`.

		### Private container registry

		@@ -316,6 +320,30 @@ The Helm post-upgrade Job reapplies the managed cluster configuration, including

		Verify the updated configuration using the checks in Section 7.

		### The OCM hub used by the CI

		The OCM hub and this chart (release `ocm-scheduler` in `hypo-scheduler-system`) are installed there by hand, once, before the first pipeline run. The hub is also used by the nightly tests.

		The merge request pipeline only tests the Go code of the scheduler:

		1. It builds the image `hypo-ocm-scheduler:<commit>`.
		2. It sets it on the Deployment `ocm-scheduler-controller-manager` with `kubectl set image`. Helm is not used, and the chart, CRD and RBAC on the hub stay unchanged.
		3. It runs the HypO OCM scenario (`@smoke-e2e-workflow-ocm`).
		4. If the tests pass, the new image stays on the hub. If they fail, the previous image is put back.

		Changes to the Helm chart, the CRD (`chart/ocm-scheduler-chart/crds/`) or the RBAC are applied to the hub by hand after the merge. Since the pipeline does not record the image in the Helm release, pass the image currently running on the hub, or Helm puts back the tag stored in the release (run from `chart/`):

		```bash
		TAG=$(kubectl get deploy ocm-scheduler-controller-manager -n hypo-scheduler-system \
		-o jsonpath="{.spec.template.spec.containers[?(@.name=='manager')].image}" \| sed 's/.*://')

		kubectl apply --server-side --force-conflicts -f ocm-scheduler-chart/crds/ # only if the CRD changed
		helm upgrade ocm-scheduler ./ocm-scheduler-chart -n hypo-scheduler-system --reuse-values \
		--set scheduler.image.tag=$TAG --wait --wait-for-jobs --timeout 10m
		```

		Helm never updates the files in `crds/` on `helm upgrade`, hence the `kubectl apply`.

		## 9. Uninstall the Scheduler

		The chart includes a Helm `pre-delete` cleanup Job that removes scheduler-specific resources before Helm deletes the release.

chart/ocm-scheduler-chart/crds/hyposchedulingmanifests.scheduling.hypo.eu.yaml

+4 −5

Changes for chart/ocm-scheduler-chart/crds/hyposchedulingmanifests.scheduling.hypo.eu.yaml: 4 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
		kind: CustomResourceDefinition
		metadata:
		annotations:
		controller-gen.kubebuilder.io/version: v0.16.1
		controller-gen.kubebuilder.io/version: v0.19.0
		name: hyposchedulingmanifests.scheduling.hypo.eu
		spec:
		group: scheduling.hypo.eu
		@@ -41,8 +41,7 @@ spec:
		metadata:
		type: object
		spec:
		description: HypoSchedulingManifestSpec defines the desired state of
		HypoSchedulingManifest
		description: HypoSchedulingManifestSpec defines the desired state of HypoSchedulingManifest
		properties:
		globalAnnotations:
		description: \|-
		@@ -86,8 +85,8 @@ spec:
		- manifests
		type: object
		status:
		description: HypoSchedulingManifestStatus defines the observed state
		of HypoSchedulingManifest
		description: HypoSchedulingManifestStatus defines the observed state of
		HypoSchedulingManifest
		properties:
		conditions:
		description: List of conditions for HypoSchedulingManifest