From fdfafbc24808ae825444f2862d91ccefd6294379 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 13:33:21 +0000 Subject: [PATCH 1/2] Correct CI/CD pipeline for QoS Profile component --- src/context/.gitlab-ci.yml | 6 +- src/qos_profile/.gitlab-ci.yml | 56 +++++++---------- .../service/QoSProfileServiceServicerImpl.py | 2 + src/qos_profile/tests/conftest.py | 60 +++++++++++++++++-- 4 files changed, 81 insertions(+), 43 deletions(-) diff --git a/src/context/.gitlab-ci.yml b/src/context/.gitlab-ci.yml index c1caed667..f0558a2d2 100644 --- a/src/context/.gitlab-ci.yml +++ b/src/context/.gitlab-ci.yml @@ -112,9 +112,9 @@ unit_test context: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy context: diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml index 0525e69b3..a688cdb05 100644 --- a/src/qos_profile/.gitlab-ci.yml +++ b/src/qos_profile/.gitlab-ci.yml @@ -15,8 +15,8 @@ # Build, tag, and push the Docker image to the GitLab Docker registry build qos_profile: variables: - IMAGE_NAME: "qos_profile" # name of the microservice - IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + IMAGE_NAME: 'qos_profile' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY @@ -30,36 +30,31 @@ build qos_profile: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml # Apply unit test to the component unit_test qos_profile: variables: - IMAGE_NAME: "qos_profile" # name of the microservice - IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + IMAGE_NAME: 'qos_profile' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: - build qos_profile before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - # QoSProfile-related - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi script: - - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - # environment preparation + - docker pull "cockroachdb/cockroach:latest-v22.2" - docker volume create crdb - > docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 @@ -72,26 +67,19 @@ unit_test qos_profile: - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - - > - # QoSProfile preparation - > docker run --name $IMAGE_NAME -d -p 3030:3030 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - # Check status before the tests - - sleep 5 - docker ps -a + - sleep 5 - docker logs $IMAGE_NAME - - # Run the tests - > docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_crud.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: # Check status after the tests @@ -102,19 +90,17 @@ unit_test qos_profile: - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force - rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml artifacts: when: always reports: diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index 47f5fbb25..e705b35ae 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -32,6 +32,8 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): LOGGER.debug('Servicer Created') self.db_engine = db_engine + def _get_metrics(self) -> MetricsPool: return METRICS_POOL + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 8983183f3..52ac56a87 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -12,18 +12,68 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest -from qos_profile.client.QoSProfileClient import QoSProfileClient +import os, pytest, sqlalchemy +from _pytest.config import Config +from _pytest.terminal import TerminalReporter +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, + get_env_var_name, get_service_port_grpc +) from common.proto.context_pb2 import Uuid, QoSProfileId from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile +from common.method_wrappers.Decorator import MetricsPool +from qos_profile.client.QoSProfileClient import QoSProfileClient +from qos_profile.service.QoSProfileService import QoSProfileService +from qos_profile.service.database.Engine import Engine +from qos_profile.service.database.models._Base import rebuild_database + +LOCAL_HOST = '127.0.0.1' +GRPC_PORT = 10000 + int(get_service_port_grpc(ServiceNameEnum.QOSPROFILE)) # avoid privileged ports + +os.environ[get_env_var_name(ServiceNameEnum.QOSPROFILE, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.QOSPROFILE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT) + +@pytest.fixture(scope='session') +def qosprofile_db(request) -> sqlalchemy.engine.Engine: # pylint: disable=unused-argument + _db_engine = Engine.get_engine() + Engine.drop_database(_db_engine) + Engine.create_database(_db_engine) + rebuild_database(_db_engine) + + _msg_broker = MessageBroker(get_messagebroker_backend()) + yield _db_engine, _msg_broker + _msg_broker.terminate() + +RAW_METRICS : MetricsPool = None + +@pytest.fixture(scope='session') +def qosprofile_service( + qosprofile_db : sqlalchemy.engine.Engine # pylint: disable=redefined-outer-name +): + global RAW_METRICS # pylint: disable=global-statement + _service = QoSProfileService(qosprofile_db) + RAW_METRICS = _service.qos_profile_servicer._get_metrics() + _service.start() + yield _service + _service.stop() -@pytest.fixture(scope='function') -def qos_profile_client(): - _client = QoSProfileClient(host='0.0.0.0', port=20040) +@pytest.fixture(scope='session') +def qos_profile_client(qosprofile_service : QoSProfileService): # pylint: disable=redefined-outer-name,unused-argument + _client = QoSProfileClient() yield _client _client.close() +@pytest.hookimpl(hookwrapper=True) +def pytest_terminal_summary( + terminalreporter : TerminalReporter, exitstatus : int, config : Config # pylint: disable=unused-argument +): + yield + if RAW_METRICS is not None: + print('') + print('Performance Results:') + print(RAW_METRICS.get_pretty_table().get_string()) def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: def create_QoSProfileValueUnitPair(data) -> QoSProfileValueUnitPair: -- GitLab From 1f558e9af78f471fffddfe515bc5a000c5fceebe Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 13:39:49 +0000 Subject: [PATCH 2/2] Correct CI/CD pipeline for QoS Profile component --- src/qos_profile/tests/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 52ac56a87..6e8d42b33 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -40,10 +40,7 @@ def qosprofile_db(request) -> sqlalchemy.engine.Engine: # pylint: disable=unus Engine.drop_database(_db_engine) Engine.create_database(_db_engine) rebuild_database(_db_engine) - - _msg_broker = MessageBroker(get_messagebroker_backend()) - yield _db_engine, _msg_broker - _msg_broker.terminate() + yield _db_engine RAW_METRICS : MetricsPool = None -- GitLab