Merge branch 'feat/318-cttc-optimize-osm-client-ci-cd-pipeline' into 'develop' (3d96577d) · Commits · TFS / controller

src/osm_client/.gitlab-ci.yml

+40 −34

Original line number	Diff line number	Diff line
		@@ -20,17 +20,14 @@ build osm_client:
		IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
		stage: build
		before_script:
		- docker image prune --force
		- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		script:
		- docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile .
		- docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		- docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		# Build mock images
		- docker buildx build -t "$MOCK_IMAGE_NAME:$IMAGE_TAG" -f ./src/tests/tools/$MOCK_IMAGE_NAME/Dockerfile .
		- docker tag "$MOCK_IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$MOCK_IMAGE_NAME:$IMAGE_TAG"
		- docker push "$CI_REGISTRY_IMAGE/$MOCK_IMAGE_NAME:$IMAGE_TAG"
		after_script:
		- docker images --filter="dangling=true" --quiet \| xargs -r docker rmi
		- docker image prune --force
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		@@ -41,8 +38,9 @@ build osm_client:
		- src/$IMAGE_NAME/Dockerfile
		- src/$IMAGE_NAME/tests/*.py
		- manifests/${IMAGE_NAME}service.yaml
		- src/tests/tools/$MOCK_IMAGE_NAME/*/.{py,in,yml}
		- src/tests/tools/$MOCK_IMAGE_NAME/Dockerfile
		- src/tests/tools/mock_osm_nbi/*/.{py,in,yml,yaml,yang,sh,json}
		- src/tests/tools/mock_osm_nbi/Dockerfile
		- src/tests/.gitlab-ci.yml
		- .gitlab-ci.yml

		# Apply unit test to the component
		@@ -54,51 +52,56 @@ unit_test osm_client:
		stage: unit_test
		needs:
		- build osm_client
		- build mock_osm_nbi
		before_script:
		# Do Docker cleanup
		- docker ps --all --quiet \| xargs --no-run-if-empty docker stop
		- docker container prune --force
		- docker ps --all --quiet \| xargs --no-run-if-empty docker rm --force
		- docker image prune --force
		- docker network prune --force
		- docker volume prune --all --force
		- docker buildx prune --force

		# Login Docker repository
		- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		- docker ps -aq \| xargs -r docker rm -f
		- >
		if docker network list \| grep teraflowbridge; then
		echo "teraflowbridge is already created";
		else
		docker network create -d bridge teraflowbridge;
		fi
		- >
		if docker container ls \| grep $IMAGE_NAME; then
		docker rm -f $IMAGE_NAME;
		else
		echo "$IMAGE_NAME image is not in the system";
		fi
		- docker container prune -f
		script:
		- docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		- docker images --filter="dangling=true" --quiet \| xargs -r docker rmi
		- docker pull "$CI_REGISTRY_IMAGE/mock-osm-nbi:test"
		- docker network create -d bridge teraflowbridge
		- >
		docker run --name $IMAGE_NAME -d -v "$PWD/src/$IMAGE_NAME/tests:/opt/results"
		docker run --name mock_osm_nbi -d
		--network=teraflowbridge
		--env LOG_LEVEL=DEBUG
		--env FLASK_ENV=development
		--env OSM_ADDRESS=mock_osm_nbi
		$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		$CI_REGISTRY_IMAGE/mock-osm-nbi:test
		- >
		docker run --name $MOCK_IMAGE_NAME -d
		docker run --name $IMAGE_NAME -d -v "$PWD/src/$IMAGE_NAME/tests:/opt/results"
		--network=teraflowbridge
		--env LOG_LEVEL=DEBUG
		--env FLASK_ENV=development
		$CI_REGISTRY_IMAGE/$MOCK_IMAGE_NAME:$IMAGE_TAG
		- while ! docker logs $IMAGE_NAME 2>&1 \| grep -q 'Configured Rules'; do sleep 1; done
		- sleep 5 # Give extra time to container to get ready
		--env OSM_ADDRESS=mock_osm_nbi
		$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		- while ! docker logs $IMAGE_NAME 2>&1 \| grep -q 'Running...'; do sleep 1; done
		- docker ps -a
		- docker logs $IMAGE_NAME
		- docker logs $MOCK_IMAGE_NAME
		- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report_unitary.xml"
		- docker logs mock_osm_nbi
		- docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report_unitary.xml"
		- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
		coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
		after_script:
		- docker logs $IMAGE_NAME
		- docker rm -f $IMAGE_NAME
		- docker rm -f $MOCK_IMAGE_NAME
		- docker network rm teraflowbridge
		- docker logs mock_osm_nbi

		# Do Docker cleanup
		- docker ps --all --quiet \| xargs --no-run-if-empty docker stop
		- docker container prune --force
		- docker ps --all --quiet \| xargs --no-run-if-empty docker rm --force
		- docker image prune --force
		- docker network prune --force
		- docker volume prune --all --force
		- docker buildx prune --force

		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		@@ -109,6 +112,9 @@ unit_test osm_client:
		- src/$IMAGE_NAME/Dockerfile
		- src/$IMAGE_NAME/tests/*.py
		- manifests/${IMAGE_NAME}service.yaml
		- src/tests/tools/mock_osm_nbi/*/.{py,in,yml,yaml,yang,sh,json}
		- src/tests/tools/mock_osm_nbi/Dockerfile
		- src/tests/.gitlab-ci.yml
		- .gitlab-ci.yml
		artifacts:
		when: always

src/osm_client/Dockerfile

+5 −3

Original line number	Diff line number	Diff line
		@@ -16,9 +16,9 @@ FROM python:3.10.16-slim


		# Install dependencies
		RUN apt-get --yes --quiet --quiet update
		RUN apt-get --yes --quiet --quiet install wget g++ git build-essential cmake make git \
		libpcre2-dev python3-dev python3-pip python3-cffi curl software-properties-common libmagic1 libmagic-dev && \
		# Unneeded: build-essential cmake libpcre2-dev python3-dev python3-pip python3-cffi curl software-properties-common libmagic-dev
		RUN apt-get --yes --quiet --quiet update && \
		apt-get --yes --quiet --quiet install wget g++ git make libmagic1 && \
		rm -rf /var/lib/apt/lists/*

		# Set Python to show logs as they occur
		@@ -62,9 +62,11 @@ WORKDIR /var/teraflow/osm_client
		ENV OSM_CLIENT_VERSION=v16.0
		RUN python3 -m pip install -r "https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=${OSM_CLIENT_VERSION}"
		RUN python3 -m pip install "git+https://osm.etsi.org/gerrit/osm/IM.git@${OSM_CLIENT_VERSION}#egg=osm-im" --upgrade

		#Clone OsmCLient code
		RUN git clone https://osm.etsi.org/gerrit/osm/osmclient
		RUN git -C osmclient checkout ${OSM_CLIENT_VERSION}

		# Install osmClient using pip
		RUN python3 -m pip install -r osmclient/requirements.txt
		RUN python3 -m pip install ./osmclient

src/osm_client/service/OsmClientServiceServicerImpl.py

+4 −1

Original line number	Diff line number	Diff line
		@@ -16,7 +16,10 @@ import grpc, logging
		from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
		from common.tools.grpc.Tools import grpc_message_to_json_string
		from common.proto.context_pb2 import (Empty)
		from common.proto.osm_client_pb2 import CreateRequest, CreateResponse, NsiListResponse, GetRequest, GetResponse, DeleteRequest, DeleteResponse
		from common.proto.osm_client_pb2 import (
		CreateRequest, CreateResponse, NsiListResponse, GetRequest, GetResponse,
		DeleteRequest, DeleteResponse
		)
		from common.proto.osm_client_pb2_grpc import OsmServiceServicer
		from osmclient import client
		from osmclient.common.exceptions import ClientException

src/osm_client/service/main.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ def main():
		grpc_service = OsmClientService()
		grpc_service.start()

		LOGGER.debug('Configured Rules:')
		LOGGER.info('Running...')

		# Wait for Ctrl+C or termination signal
		while not terminate.wait(timeout=1.0): pass

src/tests/.gitlab-ci.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -29,3 +29,4 @@ include:

		- local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml'
		- local: '/src/tests/tools/mock_qkd_node/.gitlab-ci.yml'
		- local: '/src/tests/tools/mock_osm_nbi/.gitlab-ci.yml'