From 86bdc7983c5725a2acbc1528bc5f971720319ac9 Mon Sep 17 00:00:00 2001 From: andresanaya21 Date: Fri, 22 Aug 2025 13:37:00 +0200 Subject: [PATCH] feat: add CI/CD pipeline templates for staging and deployment processes --- sdk/.gitlab-ci.yml | 158 ++++++++++++++++++ sdk/templates/ci_dev.gitlab-ci.yml | 51 ++++++ sdk/templates/ci_staging.gitlab-ci.yml | 112 +++++++++++++ .../cicd-deploy-release.gitlab-ci.yml | 29 ++++ 4 files changed, 350 insertions(+) create mode 100644 sdk/.gitlab-ci.yml create mode 100644 sdk/templates/ci_dev.gitlab-ci.yml create mode 100644 sdk/templates/ci_staging.gitlab-ci.yml create mode 100644 sdk/templates/cicd-deploy-release.gitlab-ci.yml diff --git a/sdk/.gitlab-ci.yml b/sdk/.gitlab-ci.yml new file mode 100644 index 0000000..e9016dd --- /dev/null +++ b/sdk/.gitlab-ci.yml @@ -0,0 +1,158 @@ +stages: + - test + - main_sast + - main_local_install + - main_unit_tests + - main_rf_testing + - staging_secrets_in_repo + - staging_linting + - staging_unit_tests + - staging_security + - staging_local_install + - dev_secrets_in_repo + - dev_linting + - publish_sdk_pipy + + +variables: + GITLAB_API: "https://labs.etsi.org/api/v4" + CI_DEBUG_TRACE: "true" + PROJECT_ID: "294" + SAST_EXCLUDED_ANALYZERS: "nodejs-scan" + + +.main_common: &main_common +# only: +# - merge_requests +# except: +# variables: +# - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != "OCF16-first-steps-on-ci-at-gitlab-repository" + allow_failure: true + rules: + - if: '$CI_COMMIT_REF_NAME == "main"' + when: always + - when: never + tags: + - shell + +.main_dnd: &main_dnd + allow_failure: true + services: + - docker:24.0.5-dind + rules: + - if: '$CI_COMMIT_REF_NAME == "main"' + when: always + - when: never + tags: + - docker-in-docker + +include: + - template: 'Jobs/SAST.gitlab-ci.yml' + - template: 'Jobs/Dependency-Scanning.gitlab-ci.yml' + - template: 'Jobs/Container-Scanning.gitlab-ci.yml' + - template: 'Secret-Detection.gitlab-ci.yml' + - project: 'ocf/pipeline-scripts' + ref: cicd-capif + file: + - '/capif/templates/ci_staging.gitlab-ci.yml' + - 'capif/templates/ci_dev.gitlab-ci.yml' + - 'capif/templates/ci_unit_test.gitlab-ci.yml' + - 'capif/templates/cd-deploy-ocf.gitlab-ci.yml' + - 'capif/templates/cicd-deploy-release.gitlab-ci.yml' + +semgrep-sast: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +gemnasium-python-dependency_scanning: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +gemnasium-dependency_scanning: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +secret_detection: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +container_scanning: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +# The semgrep-sast job in GitLab CI runs Semgrep static analysis rules to find +# security vulnerabilities in your source code and reports them to GitLab’s Security Dashboard. +main_semgrep_sast: + stage: main_sast + extends: semgrep-sast + variables: + DOCKER_HOST: tcp://docker:2375 + SAST_DEFAULT_ANALYZERS: bandit + <<: *main_dnd + + +# gemnasium-python-dependency_scanning → scans the dependencies for known vulnerabilities. +main_gemnasium_python_dependency_scanning: + stage: main_sast + extends: gemnasium-python-dependency_scanning + variables: + DS_ANALYZER_NAME: "gemnasium-python" + <<: *main_dnd + +# secret_detection → scans for hardcoded secrets (keys, tokens, credentials). +main_secret_detection: + stage: main_sast + extends: secret_detection + variables: + SECRET_DETECTION_HISTORIC_SCAN: "true" + <<: *main_dnd + +main_local_install: + stage: main_local_install + needs: + - main_semgrep_sast + - main_gemnasium_python_dependency_scanning + - main_secret_detection + variables: + CI_REGISTRY_USER: $CI_REGISTRY_USER + CI_REGISTRY: $CI_REGISTRY + CAPIF_DOCKER_REGISTRY: $CAPIF_DOCKER_REGISTRY + script: + - echo "# 1. Hacer un Dockerfile que haga una construya una image con la versión de código del sdk de esta rama usando este procedimiento:" + - echo "# https://labs.etsi.org/rep/ocf/sdk/-/blob/main/doc/sdk_developers.md" + - echo "# 2. Si se construye bien, que se hagan los tests unitarios" + - echo "# 3. job finalizado" + <<: *main_common + +main_unit_tests: + needs: + - main_local_install + stage: main_unit_tests + script: + - | + echo "------- Unit Tests -------" + <<: *main_common + + +main_rf_testing: + needs: ["main_unit_tests"] + stage: main_rf_testing + script: + - | + echo "------ Robot Framework Testing ------" + <<: *main_common \ No newline at end of file diff --git a/sdk/templates/ci_dev.gitlab-ci.yml b/sdk/templates/ci_dev.gitlab-ci.yml new file mode 100644 index 0000000..1243456 --- /dev/null +++ b/sdk/templates/ci_dev.gitlab-ci.yml @@ -0,0 +1,51 @@ +stages: + - dev_secrets_in_repo + - dev_linting + +variables: + CI_DEBUG_TRACE: "false" + + +.dev_common: &dev_common + tags: + - shell + +dev_secrets_in_repo: + stage: dev_secrets_in_repo + rules: + - if: '$CI_COMMIT_REF_NAME == "staging"' + when: never + - if: '$CI_COMMIT_REF_NAME == "main"' + when: never + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-release$/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always + script: + - | + pip install trufflehog + cd ../ + trufflehog sdk --exclude_paths sdk/cicd/exclusions --max_depth=5 + <<: *dev_common + +# define the process to do linting code: ruff +dev_linting_code: + stage: dev_linting + rules: + - if: '$CI_COMMIT_REF_NAME == "staging"' + when: never + - if: '$CI_COMMIT_REF_NAME == "main"' + when: never + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-release$/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always + script: + - | + echo "###ruff checks###" + pip install ruff + ruff check --config cicd/ruff.toml . || true + needs: ["dev_secrets_in_repo"] + <<: *dev_common \ No newline at end of file diff --git a/sdk/templates/ci_staging.gitlab-ci.yml b/sdk/templates/ci_staging.gitlab-ci.yml new file mode 100644 index 0000000..2d41adf --- /dev/null +++ b/sdk/templates/ci_staging.gitlab-ci.yml @@ -0,0 +1,112 @@ +stages: + - test # to Security and Compliance gitLab + - staging_secrets_in_repo + - staging_linting + - staging_security + - staging_local_install + - staging_unit_tests + +variables: + CI_DEBUG_TRACE: "false" + +.staging_common: &staging_common + only: + - merge_requests + except: + variables: + - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != "staging" + tags: + - shell + +.staging_dnd: &staging_dnd + allow_failure: true + rules: + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "staging"' + when: always + - when: never + services: + - docker:24.0.5-dind + tags: + - docker-in-docker + +staging_secrets_in_repo: + stage: staging_secrets_in_repo + script: + - | + pip install trufflehog + cd ../ + trufflehog capif --exclude_paths capif/cicd/exclusions --max_depth=5 + <<: *staging_common + +# define the process to do linting code: ruff +staging_linting_code: + stage: staging_linting + script: + - | + echo "###ruff checks###" + pip install ruff + ruff check --config cicd/ruff.toml . || true + needs: ["staging_secrets_in_repo"] + <<: *staging_common + + +# The semgrep-sast job in GitLab CI runs Semgrep static analysis rules to find +# security vulnerabilities in your source code and reports them to GitLab’s Security Dashboard. +semgrep-sast: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +# The semgrep-sast job in GitLab CI runs Semgrep static analysis rules to find +# security vulnerabilities in your source code and reports them to GitLab’s Security Dashboard. +staging_semgrep_sast: + needs: + - staging_linting_code + stage: staging_security + extends: semgrep-sast + variables: + DOCKER_HOST: tcp://docker:2375 + SAST_DEFAULT_ANALYZERS: bandit + <<: *staging_dnd + +# gemnasium-python-dependency_scanning → scans the dependencies for known vulnerabilities. +gemnasium-python-dependency_scanning: + stage: test + before_script: + - echo " ----- not run test stage -----" + rules: + - when: never + +# gemnasium-python-dependency_scanning → scans the dependencies for known vulnerabilities. +staging_gemnasium_python_sca: + needs: + - staging_linting_code + stage: staging_security + extends: gemnasium-python-dependency_scanning + variables: + DS_ANALYZER_NAME: "gemnasium-python" + <<: *staging_dnd + +staging_local_install: + needs: + - staging_gemnasium_python_sca + - staging_semgrep_sast + stage: staging_local_install + script: + - echo "# 1. Hacer un Dockerfile que haga una construya una image con la versión de código del sdk de esta rama usando este procedimiento:" + - echo "# https://labs.etsi.org/rep/ocf/sdk/-/blob/main/doc/sdk_developers.md" + - echo "# 2. Si se construye bien, que se hagan los tests unitarios" + - echo "# 3. job finalizado" + <<: *staging_common + + +staging_unit_tests: + needs: + - staging_local_install + stage: staging_unit_tests + script: + - | + echo "------- Unit Tests -------" + <<: *staging_common diff --git a/sdk/templates/cicd-deploy-release.gitlab-ci.yml b/sdk/templates/cicd-deploy-release.gitlab-ci.yml new file mode 100644 index 0000000..3aa37ed --- /dev/null +++ b/sdk/templates/cicd-deploy-release.gitlab-ci.yml @@ -0,0 +1,29 @@ +stages: + - publish_sdk_pipy + - deploy_ocf_prod + +variables: + CI_DEBUG_TRACE: "false" + CAPIF_DOCKER_REGISTRY: $CAPIF_DOCKER_REGISTRY + NAMESPACE_PROD: "ocf-prod" + DOMAIN_PROD: ocf.production + PATH_PROD: prod + +# it will only run when a new tag that starts with ‘v{major.minor.patch}-release’ is pushed +# to the repository. +.release_common: &release_common + rules: + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-release$/' + tags: + - shell + +publish_sdk_pipy: + stage: publish_sdk_pipy + rules: + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-release$/' + when: always + - when: never + tags: + - shell + script: + - echo "# 1. Procedure to publish the sdk into the pipy repository" \ No newline at end of file -- GitLab