Commit 461921fd authored by Vasilis Katopodis's avatar Vasilis Katopodis
Browse files

feat: migrate to ETSI environment

parent d341c646

.gitlab-ci.yml

+71 −38
Changes for .gitlab-ci.yml: 71 added lines, 38 removed lines.
Original line number Diff line number Diff line
@@ -9,13 +9,13 @@ stages: 
  - image_build



default:

  image: registry.ubitech.eu/nsit/maestro-apps/golang:latest

  image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/golang:latest



variables:

  DOCKER_HOST: tcp://docker:2375

  DOCKER_HOST: tcp://localhost:2375

  DOCKER_TLS_CERTDIR: ""

  DOCKER_DRIVER: overlay2

  DOCKER_API_VERSION: "1.41"

  #DOCKER_API_VERSION: "1.41"

  

  IMAGE_NAME: "pkg-manager" 

  STAG_DOCKERFILE: "build/Dockerfile"
@@ -32,7 +32,9 @@ variables: 


lint:

  stage: lint

  image: registry.ubitech.eu/nsit/maestro-apps/golangci/golangci-lint:v1.64

  image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/golangci/golangci-lint:v2.4.0

  tags:

    - hypo

  extends: .go-cache

  before_script:

    - mkdir -p ${SONAR_GO_COVERAGE}
@@ -43,7 +45,7 @@ lint: 
      - go mod download

      - |

        set -o pipefail

        golangci-lint run ./... --out-format checkstyle --print-issued-lines=true --print-linter-name=true | tee ${SONAR_GO_COVERAGE}/golanci-report.xml

        golangci-lint run ./... --output.text.path=stderr --output.checkstyle.path=stdout | tee ${SONAR_GO_COVERAGE}/golanci-report.xml

  artifacts:

    paths:

      - ${SONAR_GO_COVERAGE}
@@ -52,9 +54,11 @@ lint: 


tests_job:

  stage: test

  image: registry.ubitech.eu/nsit/maestro-apps/golang:latest

  tags:

    - hypo

  image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/golang:latest

  services:

    - name: registry.ubitech.eu/nsit/maestro-apps/docker:20.10.16-dind

    - name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16-dind

      command: [ "--tls=false" ]

      alias: docker

  before_script:
@@ -66,7 +70,8 @@ tests_job: 
    - curl -fsSL https://get.docker.com -o get-docker.sh

    - chmod +x ./get-docker.sh

    - ./get-docker.sh

    - echo -n "$MAESTRO_APPS_TOKEN" | docker login "$MAESTRO_APPS_REGISTRY" -u "$MAESTRO_APPS_USER" --password-stdin

    - echo "127.0.0.1 $(hostname)" >> /etc/hosts

    - echo "$GROUP_ACCESS_TOKEN" | docker login $HYPO_APPS_REGISTRY -u $GROUP_ACCESS_NAME --password-stdin

  script:

    - go mod download

    - gofmt -e -s -w .
@@ -80,7 +85,9 @@ tests_job: 


sonar-analysis:

  stage: analyze

  image: registry.ubitech.eu/nsit/maestro-apps/sonarsource/sonar-scanner-cli:latest

  tags:

    - hypo

  image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/sonarsource/sonar-scanner-cli:latest

  script:

    - sonar-scanner -Dsonar.projectKey=${CI_PROJECT_TITLE} -Dsonar.coverage.exclusions=**/*_test.go -Dsonar.go.coverage.reportPaths=${SONAR_GO_COVERAGE}/coverage.out -Dsonar.go.govet.reportPaths=${SONAR_GO_COVERAGE}/vet_report.json -Dsonar.go.golangci-lint.reportPaths=${SONAR_GO_COVERAGE}/golanci-report.xml

  allow_failure: true
@@ -99,49 +106,67 @@ sonar-analysis: 


dev_image_build_job:

  stage: dev_image_build

  tags:

    - hypo

  image:

    name: registry.ubitech.eu/nsit/maestro-apps/docker:20.10.16

    name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16

  services:

    - name: registry.ubitech.eu/nsit/maestro-apps/docker:20.10.16-dind

      command: [ "--tls=false" ]

    - name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16-dind

      command:

        - "--tls=false"

      alias: docker

  before_script:

    - docker info > /dev/null

    - echo -n "$MAESTRO_APPS_TOKEN" | docker login "$MAESTRO_APPS_REGISTRY" -u "$MAESTRO_APPS_USER" --password-stdin

    - docker pull registry.ubitech.eu/nsit/maestro-apps/golang:1.24

    - docker pull registry.ubitech.eu/nsit/maestro-apps/debian

    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

    - docker pull labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/golang:1.25

    - docker pull labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/debian:latest

  script:

    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

    - >

      docker build 

      --build-arg USERNAME="$CI_REGISTRY_USER" 

      --build-arg PASSWORD="$CI_JOB_TOKEN" 

      --build-arg CI_SERVER_HOST="$CI_SERVER_HOST" 

      -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA 

      -f $STAG_DOCKERFILE .

    - docker pull $CI_REGISTRY_IMAGE/$IMAGE_NAME:etsi-migration-fix

    - docker tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:etsi-migration-fix $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA

    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'



dev_deploy_job:

  stage: dev_deploy

  tags:

    - hypo

  image:

    name: registry.ubitech.eu/nsit/maestro-apps/python-argocd:3.9

    name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/python-argocd:3.9

    pull_policy: [always, if-not-present]

  before_script:

    - git clone --depth 1 --branch gitops-cd https://oauth2:$INSTALLER_TOKEN@$INSTALLER_REPO

    - mkdir -p /tmp/bin

    - export PATH="/tmp/bin:$PATH"

    - curl -L -o /tmp/age.tar.gz https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz

    - tar -xzf /tmp/age.tar.gz -C /tmp

    - mv /tmp/age/age /tmp/age/age-keygen /tmp/bin/

    - curl -L -o /tmp/bin/sops https://github.com/getsops/sops/releases/download/v3.10.2/sops-v3.10.2.linux.amd64

    - chmod +x /tmp/bin/sops

    - sops --version

    - age --version

    - git clone --depth 1 --branch gitops-cd https://oauth2:$GROUP_ACCESS_TOKEN@$INSTALLER_REPO

    - cd $INSTALLER_PATH

    - echo "$SOPS_AGE_KEY_PRIVATE" > ~/.sops-age-key.txt

    - chmod 600 ~/.sops-age-key.txt

    - export SOPS_AGE_KEY_FILE=~/.sops-age-key.txt



  script:

    - python3 scripts/helm_update.py $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA $DEV_TMF_API_IMAGE $DEV_SONATA_OSS_IMAGE $DEV_SONATA_CORE_IMAGE $DEV_ZTC_CLIENT_IMAGE $DEV_PORTAL_CART_IMAGE $DEV_TELEMETRY_IMAGE $DEV_PEERING_IMAGE $DEV_REGISTRY_IMAGE $DEV_FABRIC_IMAGE $DEV_PORTAL

    - git add .

    - git config --global user.email "$GITLAB_USER_NAME"

    - git config --global user.name "$GITLAB_USER_EMAIL"

    - sops --decrypt charts/hypo/values-cd.enc.yaml > charts/hypo/values-cd.yaml

    - python3 scripts/helm_update_etsi.py $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA $DEV_TMF_API_IMAGE $DEV_SONATA_OSS_IMAGE $DEV_SONATA_CORE_IMAGE  $DEV_PORTAL_CART_IMAGE $DEV_TELEMETRY_IMAGE $DEV_PEERING_IMAGE $DEV_REGISTRY_IMAGE $DEV_FABRIC_IMAGE $DEV_PORTAL

    - sops --encrypt --age $SOPS_AGE_KEY_PUBLIC charts/hypo/values-cd.yaml > charts/hypo/values-cd.enc.yaml

    - rm -f charts/hypo/values-cd.yaml ~/.sops-age-key.txt

    - git add charts/hypo/values-cd.enc.yaml

    - git config --global user.email "$GITLAB_USER_EMAIL"

    - git config --global user.name "$GITLAB_USER_NAME"



    - |

      if git commit -m "Update values.yaml for GitOps CD [skip ci]"; then

        git push origin gitops-cd

      else

        echo "No changes to commit."

      fi



    - |

      set -e

      argocd login "$ARGOCD_SERVER" --username "$ARGOCD_USERNAME" --password "$ARGOCD_PASSWORD" --insecure
@@ -157,23 +182,28 @@ dev_deploy_job: 


dev_workflows_job:

  stage: dev_workflows

  image: registry.ubitech.eu/nsit/maestro-apps/maven:3.9-eclipse-temurin-17

  tags:

    - hypo

  image: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/maven:3.9-eclipse-temurin-17

  before_script:

    # clone the integration tests repo

    - git clone --depth 1 "https://oauth2:$INTEGRATION_TOKEN@$INTEGRATION_REPO"

    - cd org.etsi.osl.hypo.ops.cicd.integration.tests

    - git clone --depth 1 "https://oauth2:$GROUP_ACCESS_TOKEN@$INTEGRATION_REPO"

    - cd integration.tests

  script:

    - mvn -B -q test "-Dkarate.env=staging" "-Dkarate.options=--tags @smoke-e2e-workflow-private"

    - mvn -B -q test "-Dkarate.env=$KARATE_ENV" "-Dkarate.options=--tags @smoke-e2e-workflow-private"

  dependencies:

    - dev_deploy_job

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'







semantic_versioning_job:

  image:

    name: registry.ubitech.eu/nsit/maestro-apps/node:lts

    name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/node:lts

  stage: release

  tags:

    - hypo

  variables:

    GITLAB_TOKEN: $GROUP_ACCESS_TOKEN

  before_script:
@@ -201,10 +231,12 @@ semantic_versioning_job: 


image_build_job:

  stage: image_build

  tags:

    - hypo

  image:

    name: registry.ubitech.eu/nsit/maestro-apps/docker:20.10.16

    name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16

  services:

    - name: registry.ubitech.eu/nsit/maestro-apps/docker:20.10.16-dind

    - name: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/docker:20.10.16-dind

      command: [ "--tls=false" ]

      alias: docker

  before_script:
@@ -212,9 +244,9 @@ image_build_job: 
    - RELEASE_VERSION=$(cat release_version.txt)

    - echo  $RELEASE_VERSION

    - if [ "$RELEASE_VERSION" = "no-release" ]; then exit 0; fi

    - echo -n "$MAESTRO_APPS_TOKEN" | docker login "$MAESTRO_APPS_REGISTRY" -u "$MAESTRO_APPS_USER" --password-stdin

    - docker pull registry.ubitech.eu/nsit/maestro-apps/golang:1.24

    - docker pull registry.ubitech.eu/nsit/maestro-apps/debian

    - echo "$GROUP_ACCESS_TOKEN" | docker login $HYPO_APPS_REGISTRY -u $GROUP_ACCESS_NAME --password-stdin

    - docker pull labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/golang:1.24

    - docker pull labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.ops/cicd/integration.tests/debian:latest

  script:

    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

    - >
@@ -233,3 +265,4 @@ image_build_job: 
    - if: '$CI_COMMIT_BRANCH == "main"'

  dependencies:

    - semantic_versioning_job

    
 No newline at end of file

.releaserc.json

+2 −2
Changes for .releaserc.json: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -16,12 +16,12 @@ 
          {"type": "perf", "hidden": true},

          {"type": "test", "hidden": true}

        ],

        "issueUrlFormat": "https://gitlab.ubitech.eu/maestro-v2/mdg-hypo/org.etsi.osl.hypo.core/org.etsi.osl.hypo.core.pkg-manager.helm/-/issues/{{id}}"

        "issueUrlFormat": "https://labs.etsi.org/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/-/issues/{{id}}"

      }

    }],

    "@semantic-release/release-notes-generator",

    ["@semantic-release/gitlab", {

      "gitlabUrl": "https://gitlab.ubitech.eu"

      "gitlabUrl": "https://labs.etsi.org/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm"

    }],

    ["@semantic-release/changelog",{

        "changelogFile": "./CHANGELOG.md"

CHANGELOG.md

deleted100644 → 0
+0 −510

File deleted.

Preview size limit exceeded, changes collapsed.

README.md

+2 −2
Changes for README.md: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -2,10 +2,10 @@ 


![Design Diagram](docs/design_diagram.png)



The Package Manager is a Go-based microservice responsible for the full lifecycle of Helm chart-based service deployments within Maestro.

The Package Manager is a Go-based microservice responsible for the full lifecycle of Helm chart-based service deployments within Hypo.

It bridges SONATA's orchestration workflows with Kubernetes clusters by handling chart parsing, conversion, deployment, activation/deactivation, and termination — all driven through Kafka messaging.



Maestro supports services packaged in three formats and normalises them all into Helm before deployment:

Hypo supports services packaged in three formats and normalises them all into Helm before deployment:



- Helm charts sourced directly from Helm repositories, OCI registries, or Git repositories.

- Docker-based services via [docker-compose][docker-comp-url], converted using the [Kompose][kompose-lib-url] library.

adapters/chart_converter.go

+1 −1
Changes for adapters/chart_converter.go: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ package adapters 
import (

	"strings"



	"gitlab.ubitech.eu/maestro-v2/mdg-hypo/org.etsi.osl.hypo.core/org.etsi.osl.hypo.core.pkg-manager.helm/internal/logging"

	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/internal/logging"

)



type ClusterManager string