Loading .gitlab-ci.yml +55 −16 Original line number Diff line number Diff line Loading @@ -29,7 +29,6 @@ variables: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/oss/oss-client-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/telemetry/telemetry-api-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/pkg-manager labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/service.monitor/lcm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/sonata/sonata-core-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/cart/hypo-portal-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/web/hypo-portal Loading Loading @@ -637,6 +636,8 @@ nightly_promote_images_job: - when: never nightly_compare_failed_images_job: stage: nightly_result tags: Loading @@ -661,9 +662,21 @@ nightly_compare_failed_images_job: set -eu REPORT_FILE="different_images.txt" UNAVAILABLE_FILE="unavailable_images.txt" FAILED_DATE_TAG="failed-nightly-$(date +%Y-%m-%d)" DIFFERENCES_FOUND=false cat > "$REPORT_FILE" <<EOF Nightly failure report Date: $(date +%F) The following images differed from the last successful nightly and were automatically rolled back. Backups: EOF : > "$REPORT_FILE" : > "$UNAVAILABLE_FILE" echo "Nightly workflows failed." echo "Comparing dev-latest images with latest..." Loading @@ -678,16 +691,22 @@ nightly_compare_failed_images_job: echo "Checking: $IMAGE" if ! docker pull "$DEV_IMAGE"; then echo "$IMAGE - dev-latest could not be pulled" \ >> "$REPORT_FILE" cat >> "$UNAVAILABLE_FILE" <<EOF - $(basename "$IMAGE") Reason: dev-latest could not be pulled EOF echo "Skipping rollback because dev-latest is unavailable." continue fi if ! docker pull "$LATEST_IMAGE"; then echo "$IMAGE - latest could not be pulled" \ >> "$REPORT_FILE" cat >> "$UNAVAILABLE_FILE" <<EOF - $(basename "$IMAGE") Reason: latest could not be pulled EOF echo "Skipping rollback because latest is unavailable." Loading @@ -711,16 +730,24 @@ nightly_compare_failed_images_job: if [ "$DEV_IMAGE_ID" != "$LATEST_IMAGE_ID" ]; then echo "Result: DIFFERENT" echo "$IMAGE" >> "$REPORT_FILE" # Preserve the failed dev-latest image docker tag "$DEV_IMAGE" "$FAILED_IMAGE" docker push "$FAILED_IMAGE" # Replace dev-latest with the last successful latest image # Restore dev-latest from the last successful latest image docker tag "$LATEST_IMAGE" "$DEV_IMAGE" docker push "$DEV_IMAGE" # Add it to the report only after both pushes succeed cat >> "$REPORT_FILE" <<EOF - $(basename "$IMAGE") ${FAILED_IMAGE} EOF DIFFERENCES_FOUND=true echo "Saved failed image as:" echo " $FAILED_IMAGE" Loading @@ -737,25 +764,37 @@ nightly_compare_failed_images_job: >/dev/null 2>&1 || true done if [ "$DIFFERENCES_FOUND" = false ]; then echo "No image differences were found." >> "$REPORT_FILE" echo >> "$REPORT_FILE" fi if [ -s "$UNAVAILABLE_FILE" ]; then cat >> "$REPORT_FILE" <<EOF Images that could not be checked: EOF cat "$UNAVAILABLE_FILE" >> "$REPORT_FILE" fi rm -f "$UNAVAILABLE_FILE" echo echo "Images requiring investigation:" echo "Nightly failure report:" echo "==============================================" if [ -s "$REPORT_FILE" ]; then cat "$REPORT_FILE" else echo "No differences were found." fi artifacts: when: always expire_in: 30 days paths: - different_images.txt rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_failure - when: never nightly_cleanup_job: Loading Loading
.gitlab-ci.yml +55 −16 Original line number Diff line number Diff line Loading @@ -29,7 +29,6 @@ variables: labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/oss/oss-client-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.clients/telemetry/telemetry-api-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/pkg-manager labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/service.monitor/lcm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.core/sonata/sonata-core-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/cart/hypo-portal-jvm labs.etsi.org:5050/osl/hypo/code/org.etsi.osl.hypo.portal/web/hypo-portal Loading Loading @@ -637,6 +636,8 @@ nightly_promote_images_job: - when: never nightly_compare_failed_images_job: stage: nightly_result tags: Loading @@ -661,9 +662,21 @@ nightly_compare_failed_images_job: set -eu REPORT_FILE="different_images.txt" UNAVAILABLE_FILE="unavailable_images.txt" FAILED_DATE_TAG="failed-nightly-$(date +%Y-%m-%d)" DIFFERENCES_FOUND=false cat > "$REPORT_FILE" <<EOF Nightly failure report Date: $(date +%F) The following images differed from the last successful nightly and were automatically rolled back. Backups: EOF : > "$REPORT_FILE" : > "$UNAVAILABLE_FILE" echo "Nightly workflows failed." echo "Comparing dev-latest images with latest..." Loading @@ -678,16 +691,22 @@ nightly_compare_failed_images_job: echo "Checking: $IMAGE" if ! docker pull "$DEV_IMAGE"; then echo "$IMAGE - dev-latest could not be pulled" \ >> "$REPORT_FILE" cat >> "$UNAVAILABLE_FILE" <<EOF - $(basename "$IMAGE") Reason: dev-latest could not be pulled EOF echo "Skipping rollback because dev-latest is unavailable." continue fi if ! docker pull "$LATEST_IMAGE"; then echo "$IMAGE - latest could not be pulled" \ >> "$REPORT_FILE" cat >> "$UNAVAILABLE_FILE" <<EOF - $(basename "$IMAGE") Reason: latest could not be pulled EOF echo "Skipping rollback because latest is unavailable." Loading @@ -711,16 +730,24 @@ nightly_compare_failed_images_job: if [ "$DEV_IMAGE_ID" != "$LATEST_IMAGE_ID" ]; then echo "Result: DIFFERENT" echo "$IMAGE" >> "$REPORT_FILE" # Preserve the failed dev-latest image docker tag "$DEV_IMAGE" "$FAILED_IMAGE" docker push "$FAILED_IMAGE" # Replace dev-latest with the last successful latest image # Restore dev-latest from the last successful latest image docker tag "$LATEST_IMAGE" "$DEV_IMAGE" docker push "$DEV_IMAGE" # Add it to the report only after both pushes succeed cat >> "$REPORT_FILE" <<EOF - $(basename "$IMAGE") ${FAILED_IMAGE} EOF DIFFERENCES_FOUND=true echo "Saved failed image as:" echo " $FAILED_IMAGE" Loading @@ -737,25 +764,37 @@ nightly_compare_failed_images_job: >/dev/null 2>&1 || true done if [ "$DIFFERENCES_FOUND" = false ]; then echo "No image differences were found." >> "$REPORT_FILE" echo >> "$REPORT_FILE" fi if [ -s "$UNAVAILABLE_FILE" ]; then cat >> "$REPORT_FILE" <<EOF Images that could not be checked: EOF cat "$UNAVAILABLE_FILE" >> "$REPORT_FILE" fi rm -f "$UNAVAILABLE_FILE" echo echo "Images requiring investigation:" echo "Nightly failure report:" echo "==============================================" if [ -s "$REPORT_FILE" ]; then cat "$REPORT_FILE" else echo "No differences were found." fi artifacts: when: always expire_in: 30 days paths: - different_images.txt rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_failure - when: never nightly_cleanup_job: Loading