diff --git a/proto/kpi_value.proto b/proto/kpi_value.proto new file mode 100644 index 0000000000000000000000000000000000000000..4f9f4edc5f0577f6b67f6c584936ec25377603fd --- /dev/null +++ b/proto/kpi_value.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; +package kpi_value_api; + +import "context.proto"; +import "kpi_manager.proto"; + +service KpiValueAPI { + rpc StoreKpiValues (KpiValueList) returns (context.Empty) {} + rpc SelectKpiValues (KpiValueFilter) returns (KpiValueList) {} +} + +message KpiValue { + kpi_manager.KpiId kpi_id = 1; + context.Timestamp timestamp = 2; + KpiValueType kpi_value_type = 3; +} + +message KpiValueList { + repeated KpiValue kpi_value_list = 1; +} + +message KpiValueType { + oneof value { + int32 int32Val = 1; + uint32 uint32Val = 2; + int64 int64Val = 3; + uint64 uint64Val = 4; + float floatVal = 5; + string stringVal = 6; + bool boolVal = 7; + } +} + +message KpiValueFilter { + repeated kpi_manager.KpiId kpi_id = 1; + repeated context.Timestamp start_timestamp = 2; + repeated context.Timestamp end_timestamp = 3; +} diff --git a/scripts/run_tests_locally-kpi-composer.sh b/scripts/run_tests_locally-kpi-value-API.sh similarity index 86% rename from scripts/run_tests_locally-kpi-composer.sh rename to scripts/run_tests_locally-kpi-value-API.sh index c61b257888e5d624c89b49cc4653b618e5f2c0b7..65a269ec65ac687536155e9d34755f65180ebd5c 100755 --- a/scripts/run_tests_locally-kpi-composer.sh +++ b/scripts/run_tests_locally-kpi-value-API.sh @@ -19,5 +19,5 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ - kpi_manager/tests/test_kpi_composer.py \ No newline at end of file +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_value_api/tests/test_kpi_value_api.py \ No newline at end of file diff --git a/scripts/run_tests_locally-kpi-writer.sh b/scripts/run_tests_locally-kpi-value-writer.sh similarity index 85% rename from scripts/run_tests_locally-kpi-writer.sh rename to scripts/run_tests_locally-kpi-value-writer.sh index 2bc2e51309108f8bf3d277dfe92402b07d3ff6a3..95cf396f63932342b782365560b72220dce85b78 100755 --- a/scripts/run_tests_locally-kpi-writer.sh +++ b/scripts/run_tests_locally-kpi-value-writer.sh @@ -19,5 +19,5 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ - kpi_manager/tests/test_kpi_writer.py \ No newline at end of file +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_value_writer/tests/test_kpi_value_writer.py \ No newline at end of file diff --git a/src/kpi_management/kpi_value_writer/service/__init__.py b/src/kpi_management/kpi_value_writer/service/__init__.py deleted file mode 100644 index 1549d9811aa5d1c193a44ad45d0d7773236c0612..0000000000000000000000000000000000000000 --- a/src/kpi_management/kpi_value_writer/service/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/src/kpi_management/requirements.in b/src/kpi_management/requirements.in deleted file mode 100644 index d96e4b1b8a028fee11bfb435d71b64a71747f483..0000000000000000000000000000000000000000 --- a/src/kpi_management/requirements.in +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -anytree==2.8.0 -APScheduler==3.10.1 -attrs==23.2.0 -certifi==2024.2.2 -charset-normalizer==2.0.12 -colorama==0.4.6 -confluent-kafka==2.3.0 -coverage==6.3 -future-fstrings==1.2.0 -greenlet==3.0.3 -grpcio==1.47.5 -grpcio-health-checking==1.47.5 -grpcio-tools==1.47.5 -grpclib==0.4.4 -h2==4.1.0 -hpack==4.0.0 -hyperframe==6.0.1 -idna==3.7 -influx-line-protocol==0.1.4 -iniconfig==2.0.0 -kafka-python==2.0.2 -multidict==6.0.5 -networkx==3.3 -packaging==24.0 -pluggy==1.5.0 -prettytable==3.5.0 -prometheus-client==0.13.0 -protobuf==3.20.3 -psycopg2-binary==2.9.3 -py==1.11.0 -py-cpuinfo==9.0.0 -pytest==6.2.5 -pytest-benchmark==3.4.1 -pytest-depends==1.0.1 -python-dateutil==2.8.2 -python-json-logger==2.0.2 -pytz==2024.1 -questdb==1.0.1 -requests==2.27.1 -six==1.16.0 -SQLAlchemy==1.4.52 -sqlalchemy-cockroachdb==1.4.4 -SQLAlchemy-Utils==0.38.3 -toml==0.10.2 -typing_extensions==4.12.0 -tzlocal==5.2 -urllib3==1.26.18 -wcwidth==0.2.13 -xmltodict==0.12.0 diff --git a/src/kpi_management/service/__init__.py b/src/kpi_management/service/__init__.py deleted file mode 100644 index 1549d9811aa5d1c193a44ad45d0d7773236c0612..0000000000000000000000000000000000000000 --- a/src/kpi_management/service/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/src/kpi_management/service/database/__main__.py b/src/kpi_management/service/database/__main__.py deleted file mode 100644 index 9f0e5324644a5a58eb47483688f71306a0808d1a..0000000000000000000000000000000000000000 --- a/src/kpi_management/service/database/__main__.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, signal, sys, threading, time -from prometheus_client import start_http_server -from common.Constants import ServiceNameEnum -from common.Settings import ( - ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, - wait_for_environment_variables) -from common.proto import monitoring_pb2 -from monitoring.service.EventTools import EventsDeviceCollector # import updated -from monitoring.service.NameMapping import NameMapping # import updated -# from .MonitoringService import MonitoringService -from .KpiManagerService import KpiManagerService - -terminate = threading.Event() -LOGGER = None - -def signal_handler(signal, frame): # pylint: disable=redefined-outer-name - LOGGER.warning('Terminate signal received') - terminate.set() - -def start_kpi_manager(name_mapping : NameMapping): - LOGGER.info('Start Monitoring...',) - - events_collector = EventsDeviceCollector(name_mapping) - events_collector.start() - - # TODO: redesign this method to be more clear and clean - - # Iterate while terminate is not set - while not terminate.is_set(): - list_new_kpi_ids = events_collector.listen_events() - - # Monitor Kpis - if bool(list_new_kpi_ids): - for kpi_id in list_new_kpi_ids: - # Create Monitor Kpi Requests - monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() - monitor_kpi_request.kpi_id.CopyFrom(kpi_id) - monitor_kpi_request.monitoring_window_s = 86400 - monitor_kpi_request.sampling_rate_s = 10 - events_collector._monitoring_client.MonitorKpi(monitor_kpi_request) - - time.sleep(0.5) # let other tasks run; do not overload CPU - else: - # Terminate is set, looping terminates - LOGGER.warning("Stopping execution...") - - events_collector.start() - -def main(): - global LOGGER # pylint: disable=global-statement - - log_level = get_log_level() - logging.basicConfig(level=log_level) - LOGGER = logging.getLogger(__name__) - - wait_for_environment_variables([ - get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), - get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), - get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST ), - get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), - ]) - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - LOGGER.info('Starting...') - - # Start metrics server - metrics_port = get_metrics_port() - start_http_server(metrics_port) - - name_mapping = NameMapping() - # Starting monitoring service - # grpc_service = MonitoringService(name_mapping) - # grpc_service.start() - # start_monitoring(name_mapping) - - grpc_service = KpiManagerService(name_mapping) - grpc_service.start() - - start_kpi_manager(name_mapping) - - # Wait for Ctrl+C or termination signal - while not terminate.wait(timeout=1.0): pass - - LOGGER.info('Terminating...') - grpc_service.stop() - - LOGGER.info('Bye') - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/kpi_management/tests/KPI_configs.json b/src/kpi_management/tests/KPI_configs.json deleted file mode 100644 index ba73bc41a4eccb07d8de99294c001ff7222e03f8..0000000000000000000000000000000000000000 --- a/src/kpi_management/tests/KPI_configs.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "KPIs": - [ - "node_network_receive_packets_total", - "node_network_receive_bytes_total", - "node_network_transmit_bytes_total" - ] -} \ No newline at end of file diff --git a/src/kpi_management/tests/__init__.py b/src/kpi_management/tests/__init__.py deleted file mode 100644 index 1549d9811aa5d1c193a44ad45d0d7773236c0612..0000000000000000000000000000000000000000 --- a/src/kpi_management/tests/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/src/kpi_manager/tests/test_kpi_manager.py b/src/kpi_manager/tests/test_kpi_manager.py index c746ee406b551398b22956971785bc1ffeca81c4..b517c9568fe55e5d4d83f5a1b0542605e86d3500 100755 --- a/src/kpi_manager/tests/test_kpi_manager.py +++ b/src/kpi_manager/tests/test_kpi_manager.py @@ -33,7 +33,7 @@ from device.service.driver_api.DriverInstanceCache import DriverInstanceCache from device.service.DeviceService import DeviceService from device.client.DeviceClient import DeviceClient -from kpi_manager.tests.test_messages import create_kpi_descriptor_request, create_kpi_filter_request +from kpi_manager.tests.test_messages import create_kpi_descriptor_request, create_kpi_filter_request, create_kpi_descriptor_request_a from kpi_manager.service.KpiManagerService import KpiManagerService from kpi_manager.client.KpiManagerClient import KpiManagerClient @@ -194,41 +194,48 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab ########################### # ---------- 3rd Iteration Tests ---------------- -def test_SetKpiDescriptor(kpi_manager_client): - LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") - response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) - LOGGER.info("Response gRPC message object: {:}".format(response)) - assert isinstance(response, KpiId) - -def test_DeleteKpiDescriptor(kpi_manager_client): - LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") - # adding KPI - response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) - # deleting KPI - del_response = kpi_manager_client.DeleteKpiDescriptor(response_id) - # select KPI - kpi_manager_client.GetKpiDescriptor(response_id) - LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) - assert isinstance(del_response, Empty) - -def test_GetKpiDescriptor(kpi_manager_client): - LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") - # adding KPI - response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) - # get KPI - response = kpi_manager_client.GetKpiDescriptor(response_id) - LOGGER.info("Response gRPC message object: {:}".format(response)) - assert isinstance(response, KpiDescriptor) - -def test_SelectKpiDescriptor(kpi_manager_client): - LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") - # adding KPI - kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) - # select KPI(s) - response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) - LOGGER.info("Response gRPC message object: {:}".format(response)) - assert isinstance(response, KpiDescriptorList) +# def test_SetKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiId) + +# def test_DeleteKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") +# # adding KPI +# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # deleting KPI +# del_response = kpi_manager_client.DeleteKpiDescriptor(response_id) +# # select KPI +# kpi_manager_client.GetKpiDescriptor(response_id) +# LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) +# assert isinstance(del_response, Empty) +# def test_GetKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") +# # adding KPI +# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # get KPI +# response = kpi_manager_client.GetKpiDescriptor(response_id) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiDescriptor) + +# def test_SelectKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") +# # adding KPI +# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # select KPI(s) +# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiDescriptorList) + +def test_set_list_of_KPIs(kpi_manager_client): + LOGGER.info(" >>> test_set_list_of_KPIs: START <<< ") + KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"] + # adding KPI + for kpi in KPIs_TO_SEARCH: + kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi)) + # ---------- 2nd Iteration Tests ----------------- # def test_SetKpiDescriptor(kpi_manager_client): diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py index 230cfabd06cdc796b9c264c577e05a37e845d6f0..6294d1969e71400ea8d1702314bd0cdde374341a 100644 --- a/src/kpi_manager/tests/test_messages.py +++ b/src/kpi_manager/tests/test_messages.py @@ -31,6 +31,19 @@ def create_kpi_descriptor_request(descriptor_name: str = "Test_name"): _create_kpi_request.link_id.link_uuid.uuid = 'LNK1' # pylint: disable=maybe-no-member return _create_kpi_request +def create_kpi_descriptor_request_a(description: str = "Test Description"): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_request.kpi_description = description + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member + _create_kpi_request.link_id.link_uuid.uuid = 'LNK2' # pylint: disable=maybe-no-member + return _create_kpi_request + def create_kpi_filter_request(): _create_kpi_filter_request = kpi_manager_pb2.KpiDescriptorFilter() _create_kpi_filter_request.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) diff --git a/src/kpi_management/__init__.py b/src/kpi_value_api/__init__.py similarity index 100% rename from src/kpi_management/__init__.py rename to src/kpi_value_api/__init__.py diff --git a/src/kpi_management/kpi_value_api/__init__.py b/src/kpi_value_api/client/__init__.py similarity index 100% rename from src/kpi_management/kpi_value_api/__init__.py rename to src/kpi_value_api/client/__init__.py diff --git a/src/kpi_management/kpi_value_api/client/__init__.py b/src/kpi_value_api/service/__init__.py similarity index 100% rename from src/kpi_management/kpi_value_api/client/__init__.py rename to src/kpi_value_api/service/__init__.py diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..ffd4e38ffabd0d18b85a86bca7760001a5466b73 --- /dev/null +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -0,0 +1,133 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build kpi_manager: + variables: + IMAGE_NAME: 'kpi_manager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test context: + variables: + IMAGE_NAME: 'kpi_manager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build context + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $NATS_ADDRESS + - > + docker run --name $IMAGE_NAME -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy context: +# variables: +# IMAGE_NAME: 'context' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test context +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/kpi_management/Dockerfile b/src/kpi_value_writer/Dockerfile similarity index 88% rename from src/kpi_management/Dockerfile rename to src/kpi_value_writer/Dockerfile index 0369fc0c8d9068d959a4b9a00f218b45ad3b7d46..4d74030e7b024469a236e7dfd486dcc91987cfc2 100644 --- a/src/kpi_management/Dockerfile +++ b/src/kpi_value_writer/Dockerfile @@ -54,18 +54,15 @@ RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages -RUN mkdir -p /var/teraflow/kpi_management -WORKDIR /var/teraflow/kpi_management -COPY src/kpi_management/requirements.in requirements.in +RUN mkdir -p /var/teraflow/kpi_manager +WORKDIR /var/teraflow/kpi_manager +COPY src/kpi_manager/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ -COPY src/device/. device/ -COPY src/monitoring/. monitoring/ -COPY src/kpi_management/. kpi_management/ +COPY src/kpi_manager/. kpi_manager/ # Start the service -ENTRYPOINT ["python", "-m", "kpi_management.service"] +ENTRYPOINT ["python", "-m", "kpi_manager.service"] diff --git a/src/kpi_management/README.md b/src/kpi_value_writer/README.md similarity index 70% rename from src/kpi_management/README.md rename to src/kpi_value_writer/README.md index b73f0e8af20aa6579297111d9d667b18b277ab55..72ba6e5594adeef4a29d650615716c26273ed115 100644 --- a/src/kpi_management/README.md +++ b/src/kpi_value_writer/README.md @@ -1,27 +1,29 @@ -# How to locally run and test KPI management service - -## ----- Update Required (Files structure has been changed) ----- +# How to locally run and test KPI manager micro-service +## --- File links need to be updated. --- ### Pre-requisets The following requirements should be fulfilled before the execuation of KPI management service. -1. verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. -2. virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. -3. verify the creation of required database and table. -[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiDBtests.py) python file enlist the functions to create tables and database. +1. Verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. +2. Virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. +3. Verify the creation of required database and table. +[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/database/tests/KpiDBtests.py) python file enlist the functions to create tables and database and [KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. ### Messages format templates -["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. +["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. ### Test file -["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/tests/test_kpi_management.py) python file enlist the different tests conducted during the experiment. +["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_kpi_manager.py) python file enlist different tests conducted during the experiment. ### Flow of execution (Kpi Maanager Service functions) -1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. +1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. Call `verify_tables` to verify the existence of KPI table. + 2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor in `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. + 3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from DB. + 4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches the filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. ## For KPI composer and KPI writer -The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). \ No newline at end of file +The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). \ No newline at end of file diff --git a/src/kpi_management/kpi_value_api/service/__init__.py b/src/kpi_value_writer/__init__.py similarity index 100% rename from src/kpi_management/kpi_value_api/service/__init__.py rename to src/kpi_value_writer/__init__.py diff --git a/src/kpi_management/kpi_value_writer/service/KpiValueComposer.py b/src/kpi_value_writer/service/KpiValueComposer.py similarity index 100% rename from src/kpi_management/kpi_value_writer/service/KpiValueComposer.py rename to src/kpi_value_writer/service/KpiValueComposer.py diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/kpi_management/kpi_value_writer/service/KpiWriter.py b/src/kpi_value_writer/service/KpiWriter.py similarity index 100% rename from src/kpi_management/kpi_value_writer/service/KpiWriter.py rename to src/kpi_value_writer/service/KpiWriter.py diff --git a/src/kpi_management/kpi_value_writer/__init__.py b/src/kpi_value_writer/service/__init__.py similarity index 100% rename from src/kpi_management/kpi_value_writer/__init__.py rename to src/kpi_value_writer/service/__init__.py diff --git a/src/kpi_management/service/__main__.py b/src/kpi_value_writer/service/__main__.py similarity index 96% rename from src/kpi_management/service/__main__.py rename to src/kpi_value_writer/service/__main__.py index 9f0e5324644a5a58eb47483688f71306a0808d1a..9085bc4683ba159bb64043e7b82173442f0a5bdd 100644 --- a/src/kpi_management/service/__main__.py +++ b/src/kpi_value_writer/service/__main__.py @@ -1,4 +1,4 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ from common.Settings import ( from common.proto import monitoring_pb2 from monitoring.service.EventTools import EventsDeviceCollector # import updated from monitoring.service.NameMapping import NameMapping # import updated -# from .MonitoringService import MonitoringService from .KpiManagerService import KpiManagerService terminate = threading.Event() diff --git a/src/kpi_management/tests/test_kpi_composer.py b/src/kpi_value_writer/tests/test_kpi_composer.py similarity index 100% rename from src/kpi_management/tests/test_kpi_composer.py rename to src/kpi_value_writer/tests/test_kpi_composer.py diff --git a/src/kpi_management/service/database/__init__.py b/src/kpi_value_writer/tests/test_kpi_value_writer.py old mode 100644 new mode 100755 similarity index 75% rename from src/kpi_management/service/database/__init__.py rename to src/kpi_value_writer/tests/test_kpi_value_writer.py index 1549d9811aa5d1c193a44ad45d0d7773236c0612..7b3362667b929840c481fc9c3088f8dda356bc30 --- a/src/kpi_management/service/database/__init__.py +++ b/src/kpi_value_writer/tests/test_kpi_value_writer.py @@ -12,3 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging +from kpi_manager.client.KpiManagerClient import KpiManagerClient +from kpi_value_writer.tests.test_messages import create_kpi_descriptor_request + +LOGGER = logging.getLogger(__name__) diff --git a/src/kpi_management/tests/test_kpi_writer.py b/src/kpi_value_writer/tests/test_kpi_writer.py similarity index 100% rename from src/kpi_management/tests/test_kpi_writer.py rename to src/kpi_value_writer/tests/test_kpi_writer.py diff --git a/src/kpi_management/tests/test_messages.py b/src/kpi_value_writer/tests/test_messages.py similarity index 64% rename from src/kpi_management/tests/test_messages.py rename to src/kpi_value_writer/tests/test_messages.py index 93e2d6472ada53b4430d18e6df8430ed2c3dc21e..7e59499e9332c06316b535311e4eb20e80ac7bbe 100755 --- a/src/kpi_management/tests/test_messages.py +++ b/src/kpi_value_writer/tests/test_messages.py @@ -18,29 +18,11 @@ from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.context_pb2 import DeviceId, LinkId, ServiceId, SliceId,\ ConnectionId, EndPointId -# ---------------------- 2nd iteration Test Messages --------------------------------- -def create_kpi_id_request(): - _kpi_id = kpi_manager_pb2.KpiId() - _kpi_id.kpi_id.uuid = "34f73604-eca6-424f-9995-18b519ad0978" - return _kpi_id - -def create_kpi_descriptor_request_a(descriptor_name: str = "Test_name"): - _create_kpi_request = kpi_manager_pb2.KpiDescriptor() - _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) - _create_kpi_request.kpi_description = descriptor_name - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV1' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV1' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC1' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END1' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON1' # pylint: disable=maybe-no-member - _create_kpi_request.link_id.link_uuid.uuid = 'LNK1' # pylint: disable=maybe-no-member - return _create_kpi_request - -def create_kpi_descriptor_request(): +# ---------------------- 3rd iteration Test Messages --------------------------------- +def create_kpi_descriptor_request(description: str = "Test Description"): _create_kpi_request = kpi_manager_pb2.KpiDescriptor() _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) - _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_description = description _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member @@ -50,32 +32,64 @@ def create_kpi_descriptor_request(): _create_kpi_request.link_id.link_uuid.uuid = 'LNK2' # pylint: disable=maybe-no-member return _create_kpi_request -def create_kpi_filter_request_a(): - _create_kpi_filter_request = kpi_manager_pb2.KpiDescriptorFilter() - _create_kpi_filter_request.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) - - device_id_obj = DeviceId() - endpoint_id_obj = EndPointId() - service_id_obj = ServiceId() - slice_id_obj = SliceId() - connection_id_obj = ConnectionId() - link_id_obj = LinkId() - - device_id_obj.device_uuid.uuid = "DEV1" - endpoint_id_obj.endpoint_uuid.uuid = "END1" - service_id_obj.service_uuid.uuid = "SERV1" - slice_id_obj.slice_uuid.uuid = "SLC1" - connection_id_obj.connection_uuid.uuid = "CON1" - link_id_obj.link_uuid.uuid = "LNK1" - - _create_kpi_filter_request.device_id.append(device_id_obj) - _create_kpi_filter_request.endpoint_id.append(endpoint_id_obj) - _create_kpi_filter_request.service_id.append(service_id_obj) - _create_kpi_filter_request.slice_id.append(slice_id_obj) - _create_kpi_filter_request.connection_id.append(connection_id_obj) - _create_kpi_filter_request.link_id.append(link_id_obj) - - return _create_kpi_filter_request +# ---------------------- 2nd iteration Test Messages --------------------------------- +# def create_kpi_id_request(): +# _kpi_id = kpi_manager_pb2.KpiId() +# _kpi_id.kpi_id.uuid = "34f73604-eca6-424f-9995-18b519ad0978" +# return _kpi_id + +# def create_kpi_descriptor_request_a(descriptor_name: str = "Test_name"): +# _create_kpi_request = kpi_manager_pb2.KpiDescriptor() +# _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) +# _create_kpi_request.kpi_description = descriptor_name +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV1' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV1' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC1' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END1' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON1' # pylint: disable=maybe-no-member +# _create_kpi_request.link_id.link_uuid.uuid = 'LNK1' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_descriptor_request(): +# _create_kpi_request = kpi_manager_pb2.KpiDescriptor() +# _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member +# _create_kpi_request.link_id.link_uuid.uuid = 'LNK2' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_filter_request_a(): +# _create_kpi_filter_request = kpi_manager_pb2.KpiDescriptorFilter() +# _create_kpi_filter_request.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) + +# device_id_obj = DeviceId() +# endpoint_id_obj = EndPointId() +# service_id_obj = ServiceId() +# slice_id_obj = SliceId() +# connection_id_obj = ConnectionId() +# link_id_obj = LinkId() + +# device_id_obj.device_uuid.uuid = "DEV1" +# endpoint_id_obj.endpoint_uuid.uuid = "END1" +# service_id_obj.service_uuid.uuid = "SERV1" +# slice_id_obj.slice_uuid.uuid = "SLC1" +# connection_id_obj.connection_uuid.uuid = "CON1" +# link_id_obj.link_uuid.uuid = "LNK1" + +# _create_kpi_filter_request.device_id.append(device_id_obj) +# _create_kpi_filter_request.endpoint_id.append(endpoint_id_obj) +# _create_kpi_filter_request.service_id.append(service_id_obj) +# _create_kpi_filter_request.slice_id.append(slice_id_obj) +# _create_kpi_filter_request.connection_id.append(connection_id_obj) +# _create_kpi_filter_request.link_id.append(link_id_obj) + +# return _create_kpi_filter_request # -------------------- Initial Test messages -------------------------------------