From ee3d0767356b3b7c198fd28d4737bc97a8eaa72d Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 11 Mar 2024 08:43:30 +0000 Subject: [PATCH 001/106] Changes are made to activate monitoring module, e2e Orchestrator, enable drop cocDB --- my_deploy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index 0fcb51f90..5bd58d0dc 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -23,7 +23,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" # Uncomment to activate Monitoring -#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate ZTP #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" @@ -44,7 +44,7 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene #export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" # Uncomment to activate E2E Orchestrator -#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" @@ -93,7 +93,7 @@ export CRDB_DATABASE="tfs" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="" +export CRDB_DROP_DATABASE_IF_EXISTS="YES" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" @@ -141,7 +141,7 @@ export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" # Disable flag for dropping tables if they exist. -export QDB_DROP_TABLES_IF_EXIST="" +export QDB_DROP_TABLES_IF_EXIST="YES" # Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" -- GitLab From c67611f30b5fce1d88e718e89288810a7c5a6e73 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:07:36 +0000 Subject: [PATCH 002/106] New kpi_manger.proto file is created. --- proto/kpi_manager.proto | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 proto/kpi_manager.proto diff --git a/proto/kpi_manager.proto b/proto/kpi_manager.proto new file mode 100644 index 000000000..f5769ed37 --- /dev/null +++ b/proto/kpi_manager.proto @@ -0,0 +1,47 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_manager; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service KpiManagerService{ + rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final + rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final + rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + repeated KpiId kpi_id_list = 3; + kpi_sample_types.KpiSampleType kpi_sample_type = 4; + context.DeviceId device_id = 5; + context.EndPointId endpoint_id = 6; + context.ServiceId service_id = 7; + context.SliceId slice_id = 8; + context.ConnectionId connection_id = 9; + context.LinkId link_id = 10; +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} \ No newline at end of file -- GitLab From 5a85e8f320c5b0e9492436c2a9c09f6f8ba25074 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:08:52 +0000 Subject: [PATCH 003/106] imports are updated to refer to kpi_manager.proto --- proto/device.proto | 7 ++++--- proto/optical_attack_detector.proto | 5 +++-- proto/policy_condition.proto | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/proto/device.proto b/proto/device.proto index 30e60079d..98cca8ce9 100644 --- a/proto/device.proto +++ b/proto/device.proto @@ -16,7 +16,8 @@ syntax = "proto3"; package device; import "context.proto"; -import "monitoring.proto"; +//import "monitoring.proto"; +import "kpi_manager.proto"; service DeviceService { rpc AddDevice (context.Device ) returns (context.DeviceId ) {} @@ -27,8 +28,8 @@ service DeviceService { } message MonitoringSettings { - monitoring.KpiId kpi_id = 1; - monitoring.KpiDescriptor kpi_descriptor = 2; + kpi_manager.KpiId kpi_id = 1; + kpi_manager.KpiDescriptor kpi_descriptor = 2; float sampling_duration_s = 3; float sampling_interval_s = 4; } diff --git a/proto/optical_attack_detector.proto b/proto/optical_attack_detector.proto index ebe3b5e06..0d3ed58de 100644 --- a/proto/optical_attack_detector.proto +++ b/proto/optical_attack_detector.proto @@ -17,7 +17,8 @@ syntax = "proto3"; package optical_attack_detector; import "context.proto"; -import "monitoring.proto"; +//import "monitoring.proto"; +import "kpi_manager.proto"; service OpticalAttackDetectorService { @@ -28,5 +29,5 @@ service OpticalAttackDetectorService { message DetectionRequest { context.ServiceId service_id = 1; - monitoring.KpiId kpi_id = 2; + kpi_manager.KpiId kpi_id = 2; } diff --git a/proto/policy_condition.proto b/proto/policy_condition.proto index 2037af93c..c0af929ef 100644 --- a/proto/policy_condition.proto +++ b/proto/policy_condition.proto @@ -16,10 +16,11 @@ syntax = "proto3"; package policy; import "monitoring.proto"; +import "kpi_manager.proto"; // Condition message PolicyRuleCondition { - monitoring.KpiId kpiId = 1; + kpi_manager.KpiId kpiId = 1; NumericalOperator numericalOperator = 2; monitoring.KpiValue kpiValue = 3; } -- GitLab From a78bcae188ec444a5e78f8a30df6488814d26176 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:10:14 +0000 Subject: [PATCH 004/106] few methods and messages are moved to kpi_manager.ptoto --- proto/monitoring.proto | 58 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) mode change 100644 => 100755 proto/monitoring.proto diff --git a/proto/monitoring.proto b/proto/monitoring.proto old mode 100644 new mode 100755 index 45ba48b02..2706988aa --- a/proto/monitoring.proto +++ b/proto/monitoring.proto @@ -16,13 +16,14 @@ syntax = "proto3"; package monitoring; import "context.proto"; -import "kpi_sample_types.proto"; +import "kpi_manager.proto"; +//import "kpi_sample_types.proto"; service MonitoringService { - rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final - rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final - rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final - rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final +// rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final +// rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final +// rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final +// rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final rpc IncludeKpi (Kpi ) returns (context.Empty ) {} // Stable and final rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {} // Stable and final rpc QueryKpiData (KpiQuery ) returns (RawKpiTable ) {} // Not implemented @@ -35,36 +36,25 @@ service MonitoringService { rpc GetAlarmDescriptor (AlarmID ) returns (AlarmDescriptor ) {} // Stable and final rpc GetAlarmResponseStream(AlarmSubscription ) returns (stream AlarmResponse) {} // Not Stable not final rpc DeleteAlarm (AlarmID ) returns (context.Empty ) {} // Stable and final - rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final - rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final +// rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final +// rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final } -message KpiDescriptor { - KpiId kpi_id = 1; - string kpi_description = 2; - repeated KpiId kpi_id_list = 3; - kpi_sample_types.KpiSampleType kpi_sample_type = 4; - context.DeviceId device_id = 5; - context.EndPointId endpoint_id = 6; - context.ServiceId service_id = 7; - context.SliceId slice_id = 8; - context.ConnectionId connection_id = 9; - context.LinkId link_id = 10; -} + message MonitorKpiRequest { - KpiId kpi_id = 1; + kpi_manager.KpiId kpi_id = 1; float monitoring_window_s = 2; float sampling_rate_s = 3; // Pending add field to reflect Available Device Protocols } message KpiQuery { - repeated KpiId kpi_ids = 1; - float monitoring_window_s = 2; - uint32 last_n_samples = 3; // used when you want something like "get the last N many samples - context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" - context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" + repeated kpi_manager.KpiId kpi_ids = 1; + float monitoring_window_s = 2; + uint32 last_n_samples = 3; // used when you want something like "get the last N many samples + context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" } @@ -74,20 +64,18 @@ message RawKpi { // cell } message RawKpiList { // column - KpiId kpi_id = 1; - repeated RawKpi raw_kpis = 2; + kpi_manager.KpiId kpi_id = 1; + repeated RawKpi raw_kpis = 2; } message RawKpiTable { // table repeated RawKpiList raw_kpi_lists = 1; } -message KpiId { - context.Uuid kpi_id = 1; -} + message Kpi { - KpiId kpi_id = 1; + kpi_manager.KpiId kpi_id = 1; context.Timestamp timestamp = 2; KpiValue kpi_value = 3; } @@ -117,13 +105,11 @@ message KpiList { repeated Kpi kpi = 1; } -message KpiDescriptorList { - repeated KpiDescriptor kpi_descriptor_list = 1; -} + message SubsDescriptor{ SubscriptionID subs_id = 1; - KpiId kpi_id = 2; + kpi_manager.KpiId kpi_id = 2; float sampling_duration_s = 3; float sampling_interval_s = 4; context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time" @@ -148,7 +134,7 @@ message AlarmDescriptor { AlarmID alarm_id = 1; string alarm_description = 2; string name = 3; - KpiId kpi_id = 4; + kpi_manager.KpiId kpi_id = 4; KpiValueRange kpi_value_range = 5; context.Timestamp timestamp = 6; } -- GitLab From c02883ed78e500104a4f57cdbd2f9aa9f7315cc4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:11:13 +0000 Subject: [PATCH 005/106] service name enum and default grpc port is added --- src/common/Constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/Constants.py b/src/common/Constants.py index 30aa09b4c..838b028a7 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -43,6 +43,7 @@ class ServiceNameEnum(Enum): ZTP = 'ztp' POLICY = 'policy' MONITORING = 'monitoring' + KPIMANGER = 'kpiManager' DLT = 'dlt' NBI = 'nbi' CYBERSECURITY = 'cybersecurity' @@ -73,6 +74,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ZTP .value : 5050, ServiceNameEnum.POLICY .value : 6060, ServiceNameEnum.MONITORING .value : 7070, + ServiceNameEnum.KPIMANGER .value : 7071, ServiceNameEnum.DLT .value : 8080, ServiceNameEnum.NBI .value : 9090, ServiceNameEnum.L3_CAD .value : 10001, -- GitLab From 65fd027c946f5e266311877f0541e030867fd1ee Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:11:59 +0000 Subject: [PATCH 006/106] initial client file for kpi manager --- src/kpi_manager/client/KpiManagerClient.py | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/kpi_manager/client/KpiManagerClient.py diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py new file mode 100644 index 000000000..3aa6fc65d --- /dev/null +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -0,0 +1,76 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.proto.context_pb2 import Empty +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList, +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class KpiManagerclient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.KPIMANGER) # update enum + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # update enum + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = KpiManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def SetKpi(self, request : KpiDescriptor) -> KpiId: + LOGGER.debug('SetKpi: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SetKpi(request) + LOGGER.debug('SetKpi result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteKpi(self,request : KpiId) -> Empty: + LOGGER.debug('DeleteKpi: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteKpi(request) + LOGGER.info('DeleteKpi result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetKpiDescriptor(self, request : KpiId) -> KpiDescriptor: + LOGGER.debug('GetKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetKpiDescriptor(request) + LOGGER.debug('GetKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetKpiDescriptorList(self, request : Empty) -> KpiDescriptorList: + LOGGER.debug('GetKpiDescriptorList: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetKpiDescriptorList(request) + LOGGER.debug('GetKpiDescriptorList result: {:s}'.format(grpc_message_to_json_string(response))) + return response \ No newline at end of file -- GitLab From 0b5fd79d8270475cf0e38cc3970ad77a14e8a08e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:13:09 +0000 Subject: [PATCH 007/106] initial service file for kpi manager --- src/kpi_manager/server/KpiManagerServer.py | 122 +++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/kpi_manager/server/KpiManagerServer.py diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py new file mode 100644 index 000000000..0a8932aab --- /dev/null +++ b/src/kpi_manager/server/KpiManagerServer.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from monitoring.service.NameMapping import NameMapping + +from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList +from monitoring.service import ManagementDBTools + +from common.tools.service.GenericGrpcService import GenericGrpcService + +LOGGER = logging.getLogger(__name__) + +METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") +METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT") +METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT") +METRICSDB_TABLE_MONITORING_KPIS = os.environ.get("METRICSDB_TABLE_MONITORING_KPIS") + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServer(KpiManagerServiceServicer): + def __init__(self, cls_name: str = __name__): + LOGGER.info('Init KpiManagerService') + port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # port updated + GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + def install_servicers(self): + # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class + # implementation exists in the same class. + add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From 5b6399dc2bb9665bbcfc4207239923f82549d938 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:36:20 +0000 Subject: [PATCH 008/106] TYPO - "KPIMANGER" changed to "KPIMANAGER" --- src/common/Constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Constants.py b/src/common/Constants.py index 838b028a7..4a0f3a226 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -43,7 +43,7 @@ class ServiceNameEnum(Enum): ZTP = 'ztp' POLICY = 'policy' MONITORING = 'monitoring' - KPIMANGER = 'kpiManager' + KPIMANAGER = 'kpiManager' DLT = 'dlt' NBI = 'nbi' CYBERSECURITY = 'cybersecurity' -- GitLab From 67146f1e6061f3190659ef1d0f87e07b270c746c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:37:23 +0000 Subject: [PATCH 009/106] updated KPIMANAGER ServiceEnumName --- src/kpi_manager/client/KpiManagerClient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index 3aa6fc65d..d31a8b60f 100644 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -29,8 +29,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class KpiManagerclient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.KPIMANGER) # update enum - if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # update enum + if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) # update enum + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # update enum self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) self.channel = None -- GitLab From 2223326625edeeab4a89e0535ce490e73725db1d Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:38:50 +0000 Subject: [PATCH 010/106] updation of KPIMANAGER ServiceEnumName and removal of unnecssary variables. --- src/kpi_manager/server/KpiManagerServer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py index 0a8932aab..d42ce14eb 100644 --- a/src/kpi_manager/server/KpiManagerServer.py +++ b/src/kpi_manager/server/KpiManagerServer.py @@ -29,17 +29,12 @@ from common.tools.service.GenericGrpcService import GenericGrpcService LOGGER = logging.getLogger(__name__) -METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") -METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT") -METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT") -METRICSDB_TABLE_MONITORING_KPIS = os.environ.get("METRICSDB_TABLE_MONITORING_KPIS") - METRICS_POOL = MetricsPool('Monitoring', 'RPC') class KpiManagerServer(KpiManagerServiceServicer): def __init__(self, cls_name: str = __name__): LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # port updated + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed # Init sqlite monitoring db -- GitLab From e5bfc7b10d25c24d03ecc348d948dba4d0639529 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:21:45 +0000 Subject: [PATCH 011/106] blank line is removed --- src/kpi_manager/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/__init__.py diff --git a/src/kpi_manager/__init__.py b/src/kpi_manager/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From 7ef9ed5c57db3a562f10453958baf18cff6c5ef0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:23:44 +0000 Subject: [PATCH 012/106] Class name is changed from "KpiManagerclient" to "KpiManagerClient" --- src/kpi_manager/client/KpiManagerClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index d31a8b60f..6baca7cef 100644 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -27,7 +27,7 @@ MAX_RETRIES = 15 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') -class KpiManagerclient: +class KpiManagerClient: def __init__(self, host=None, port=None): if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) # update enum if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # update enum -- GitLab From e6b8b7a57ad254ae12bf705238f6b8d6e72c01af Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:24:11 +0000 Subject: [PATCH 013/106] blank line is removed --- src/kpi_manager/service/KpiManagerService.py | 117 +++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/kpi_manager/service/KpiManagerService.py diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py new file mode 100644 index 000000000..d42ce14eb --- /dev/null +++ b/src/kpi_manager/service/KpiManagerService.py @@ -0,0 +1,117 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from monitoring.service.NameMapping import NameMapping + +from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList +from monitoring.service import ManagementDBTools + +from common.tools.service.GenericGrpcService import GenericGrpcService + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServer(KpiManagerServiceServicer): + def __init__(self, cls_name: str = __name__): + LOGGER.info('Init KpiManagerService') + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated + GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + def install_servicers(self): + # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class + # implementation exists in the same class. + add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From 90ffb7281d4db0a987d1902640949ad4153f0462 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:25:57 +0000 Subject: [PATCH 014/106] KpiManager test messages file initial version. --- src/kpi_manager/tests/test_messages.py | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/kpi_manager/tests/test_messages.py diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py new file mode 100644 index 000000000..589d6cb84 --- /dev/null +++ b/src/kpi_manager/tests/test_messages.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.proto import kpi_manager_pb2 +from common.proto.kpi_sample_types_pb2 import KpiSampleType + +def kpi_id(): + _kpi_id = kpi_manager_pb2.KpiId() + _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member + return _kpi_id + +def create_kpi_request(kpi_id_str): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) + _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) + return _create_kpi_request + +def create_kpi_request_b(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member + return _create_kpi_request + +def create_kpi_request_c(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member + return _create_kpi_request + +def create_kpi_request_d(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member + return _create_kpi_request + +def kpi_descriptor_list(): + _kpi_descriptor_list = kpi_manager_pb2.KpiDescriptorList() + return _kpi_descriptor_list \ No newline at end of file -- GitLab From 7fdc29ac02f3a8dc0f35b5fb1d8cca0af3edf4da Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:28:35 +0000 Subject: [PATCH 015/106] The methods moved to KpiManagerMessages file are removed. --- src/monitoring/tests/Messages.py | 102 +++++++++++++++---------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/src/monitoring/tests/Messages.py b/src/monitoring/tests/Messages.py index a56207d9a..23f4db017 100644 --- a/src/monitoring/tests/Messages.py +++ b/src/monitoring/tests/Messages.py @@ -17,54 +17,54 @@ from common.proto import monitoring_pb2 from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.tools.timestamp.Converters import timestamp_utcnow_to_float -def kpi_id(): - _kpi_id = monitoring_pb2.KpiId() - _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member - return _kpi_id - -def create_kpi_request(kpi_id_str): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) - _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) - return _create_kpi_request - -def create_kpi_request_b(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member - return _create_kpi_request - -def create_kpi_request_c(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member - return _create_kpi_request - -def create_kpi_request_d(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member - return _create_kpi_request +# def kpi_id(): +# _kpi_id = monitoring_pb2.KpiId() +# _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member +# return _kpi_id + +# def create_kpi_request(kpi_id_str): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) +# return _create_kpi_request + +# def create_kpi_request_b(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_request_c(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_request_d(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member +# return _create_kpi_request def monitor_kpi_request(kpi_uuid, monitoring_window_s, sampling_rate_s): _monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() @@ -80,10 +80,10 @@ def include_kpi_request(kpi_id): _include_kpi_request.kpi_value.floatVal = 500*random() # pylint: disable=maybe-no-member return _include_kpi_request -def kpi_descriptor_list(): - _kpi_descriptor_list = monitoring_pb2.KpiDescriptorList() +# def kpi_descriptor_list(): +# _kpi_descriptor_list = monitoring_pb2.KpiDescriptorList() - return _kpi_descriptor_list +# return _kpi_descriptor_list def kpi_query(kpi_id_list): _kpi_query = monitoring_pb2.KpiQuery() -- GitLab From 7386b351cb42c83fd428d35129dbb44a54e5bbaf Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:29:55 +0000 Subject: [PATCH 016/106] Blank line from the top is removed. --- src/kpi_manager/tests/test_unitary.py | 267 ++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 src/kpi_manager/tests/test_unitary.py diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py new file mode 100644 index 000000000..39d2b2874 --- /dev/null +++ b/src/kpi_manager/tests/test_unitary.py @@ -0,0 +1,267 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, pytest +import logging, json + +from apscheduler.schedulers.background import BackgroundScheduler + +from common.proto.context_pb2 import ConfigActionEnum, Context, ContextId, DeviceOperationalStatusEnum, EventTypeEnum, DeviceEvent, Device, Empty, Topology, TopologyId +from common.Constants import ServiceNameEnum +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME, ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.object_factory.Context import json_context, json_context_id +from common.tools.object_factory.Topology import json_topology, json_topology_id +# from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ +# AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList + + +from device.service.driver_api.DriverFactory import DriverFactory +from device.service.driver_api.DriverInstanceCache import DriverInstanceCache +from device.service.DeviceService import DeviceService +from device.client.DeviceClient import DeviceClient + +from kpi_manager.tests.test_messages import create_kpi_request, create_kpi_request_b, create_kpi_request_c, create_kpi_request_d +# from monitoring.service.MonitoringService import MonitoringService +from kpi_manager.service.KpiManagerService import KpiManagerService +# from monitoring.client.MonitoringClient import MonitoringClient +from kpi_manager.client.KpiManagerClient import KpiManagerClient + +from monitoring.service.ManagementDBTools import ManagementDB +from monitoring.service.MetricsDBTools import MetricsDB +from monitoring.service.NameMapping import NameMapping + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 + +KPIMANAGER_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # avoid privileged ports +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(KPIMANAGER_SERVICE_PORT) + +METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME') + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of Monitoring + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +@pytest.fixture(scope='session') +def context_service(): + LOGGER.info('Initializing MockContextService...') + _service = MockContextService(MOCKSERVICE_PORT) + _service.start() + + LOGGER.info('Yielding MockContextService...') + yield _service + + LOGGER.info('Terminating MockContextService...') + _service.context_servicer.msg_broker.terminate() + _service.stop() + + LOGGER.info('Terminated MockContextService...') + +@pytest.fixture(scope='session') +def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing ContextClient...') + _client = ContextClient() + + LOGGER.info('Yielding ContextClient...') + yield _client + + LOGGER.info('Closing ContextClient...') + _client.close() + + LOGGER.info('Closed ContextClient...') + +@pytest.fixture(scope='session') +def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceService...') + driver_factory = DriverFactory(DRIVERS) + driver_instance_cache = DriverInstanceCache(driver_factory) + _service = DeviceService(driver_instance_cache) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding DeviceService...') + yield _service + + LOGGER.info('Terminating DeviceService...') + _service.stop() + + LOGGER.info('Terminated DeviceService...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +# This fixture will be requested by test cases and last during testing session +@pytest.fixture(scope='session') +def kpi_manager_service( + context_service : MockContextService, # pylint: disable=redefined-outer-name,unused-argument + device_service : DeviceService # pylint: disable=redefined-outer-name,unused-argument + ): + LOGGER.info('Initializing KpiManagerService...') + name_mapping = NameMapping() + # _service = MonitoringService(name_mapping) + _service = KpiManagerService(name_mapping) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerService...') + yield _service + + LOGGER.info('Terminating KpiManagerService...') + _service.stop() + + LOGGER.info('Terminated KpiManagerService...') + +# This fixture will be requested by test cases and last during testing session. +# The client requires the server, so client fixture has the server as dependency. +# def monitoring_client(monitoring_service : MonitoringService): (Add for better understanding) +@pytest.fixture(scope='session') +def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing KpiManagerClient...') + _client = KpiManagerClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerClient...') + yield _client + + LOGGER.info('Closing KpiManagerClient...') + _client.close() + + LOGGER.info('Closed KpiManagerClient...') + +@pytest.fixture(scope='session') +def management_db(): + _management_db = ManagementDB('monitoring.db') + return _management_db + +@pytest.fixture(scope='session') +def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name + return monitoring_service.monitoring_servicer.metrics_db + +# This function os not clear to me (Changes should me made before execution) +@pytest.fixture(scope='session') +def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name + return monitoring_service.monitoring_servicer.metrics_db + #_metrics_db = MetricsDBTools.MetricsDB( + # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) + #return _metrics_db + +@pytest.fixture(scope='session') +def subs_scheduler(): + _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) + _scheduler.start() + return _scheduler + +def ingestion_data(kpi_id_int): + # pylint: disable=redefined-outer-name,unused-argument + metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') + + kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') + for _ in range(50): + kpiSampleType = kpiSampleType_name + kpiId = kpi_id_int + deviceId = 'DEV'+ str(kpi_id_int) + endpointId = 'END' + str(kpi_id_int) + serviceId = 'SERV' + str(kpi_id_int) + sliceId = 'SLC' + str(kpi_id_int) + connectionId = 'CON' + str(kpi_id_int) + time_stamp = timestamp_utcnow_to_float() + kpi_value = 500*random() + + metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, + kpi_value) + sleep(0.1) + +################################################## +# Prepare Environment, should be the first test +################################################## + +def test_prepare_environment( + context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument +): + context_id = json_context_id(DEFAULT_CONTEXT_NAME) + context_client.SetContext(Context(**json_context(DEFAULT_CONTEXT_NAME))) + context_client.SetTopology(Topology(**json_topology(DEFAULT_TOPOLOGY_NAME, context_id=context_id))) + +########################### +# Tests Implementation +########################### + +# Test case that makes use of client fixture to test server's CreateKpi method +def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name + # make call to server + LOGGER.warning('test_create_kpi requesting') + for i in range(3): + response = kpi_manager_client.SetKpi(create_kpi_request(str(i+1))) + LOGGER.debug(str(response)) + assert isinstance(response, KpiId) + +# Test case that makes use of client fixture to test server's DeleteKpi method +def test_delete_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name + # make call to server + LOGGER.warning('delete_kpi requesting') + response = kpi_manager_client.SetKpi(create_kpi_request('4')) + response = kpi_manager_client.DeleteKpi(response) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +# Test case that makes use of client fixture to test server's GetKpiDescriptor method +def test_get_kpi_descriptor_list(kpi_manager_client): # pylint: disable=redefined-outer-name + LOGGER.warning('test_getkpidescritor_kpi begin') + response = kpi_manager_client.GetKpiDescriptorList(Empty()) + LOGGER.debug(str(response)) + assert isinstance(response, KpiDescriptorList) -- GitLab From 124d09dec46e2477c23304605e3628200efc84a7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:30:45 +0000 Subject: [PATCH 017/106] File name changed from "KpiManagerServer" to "KpiManagerService" --- src/kpi_manager/server/KpiManagerServer.py | 117 --------------------- 1 file changed, 117 deletions(-) delete mode 100644 src/kpi_manager/server/KpiManagerServer.py diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py deleted file mode 100644 index d42ce14eb..000000000 --- a/src/kpi_manager/server/KpiManagerServer.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, os, grpc -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty - -from common.Constants import ServiceNameEnum -from common.Settings import get_service_port_grpc -from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server -from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer -from monitoring.service.NameMapping import NameMapping - -from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList -from monitoring.service import ManagementDBTools - -from common.tools.service.GenericGrpcService import GenericGrpcService - -LOGGER = logging.getLogger(__name__) - -METRICS_POOL = MetricsPool('Monitoring', 'RPC') - -class KpiManagerServer(KpiManagerServiceServicer): - def __init__(self, cls_name: str = __name__): - LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated - GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed - - # Init sqlite monitoring db - self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? - LOGGER.info('MetricsDB initialized --- KPI Manager Service') - - def install_servicers(self): - # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class - # implementation exists in the same class. - add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SetKpi( - self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: - response = KpiId() - kpi_description = request.kpi_description - kpi_sample_type = request.kpi_sample_type - kpi_device_id = request.device_id.device_uuid.uuid - kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid - kpi_service_id = request.service_id.service_uuid.uuid - kpi_slice_id = request.slice_id.slice_uuid.uuid - kpi_connection_id = request.connection_id.connection_uuid.uuid - kpi_link_id = request.link_id.link_uuid.uuid - if request.kpi_id.kpi_id.uuid != "": - response.kpi_id.uuid = request.kpi_id.kpi_id.uuid - # Here the code to modify an existing kpi - else: - data = self.management_db.insert_KPI( - kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, - kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) - response.kpi_id.uuid = str(data) - return response - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: - kpi_id = int(request.kpi_id.uuid) - kpi = self.management_db.get_KPI(kpi_id) - if kpi: - self.management_db.delete_KPI(kpi_id) - else: - LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) - return Empty() - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: - kpi_id = request.kpi_id.uuid - kpi_db = self.management_db.get_KPI(int(kpi_id)) - kpiDescriptor = KpiDescriptor() - if kpi_db is None: - LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) - else: - kpiDescriptor.kpi_description = kpi_db[1] - kpiDescriptor.kpi_sample_type = kpi_db[2] - kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) - kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) - kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) - kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) - kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) - kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) - return kpiDescriptor - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: - kpi_descriptor_list = KpiDescriptorList() - data = self.management_db.get_KPIS() - LOGGER.debug(f"data: {data}") - for item in data: - kpi_descriptor = KpiDescriptor() - kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) - kpi_descriptor.kpi_description = item[1] - kpi_descriptor.kpi_sample_type = item[2] - kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) - kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) - kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) - kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) - kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) - kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) - kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) - return kpi_descriptor_list \ No newline at end of file -- GitLab From 71b8b5362b17711d2a4bc2eeaf0c5c17b7a2b827 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:33:29 +0000 Subject: [PATCH 018/106] imports of KpiId-Descriptor-List is changed from monitoring_pb2 to kpi_manger_pb2 --- src/monitoring/client/MonitoringClient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/monitoring/client/MonitoringClient.py b/src/monitoring/client/MonitoringClient.py index 751ff6e38..493e96ca8 100644 --- a/src/monitoring/client/MonitoringClient.py +++ b/src/monitoring/client/MonitoringClient.py @@ -20,8 +20,9 @@ from common.Settings import get_service_host, get_service_port_grpc from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.monitoring_pb2 import Kpi, KpiDescriptor, KpiId, MonitorKpiRequest, \ - KpiDescriptorList, KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsList, \ +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.proto.monitoring_pb2 import Kpi, MonitorKpiRequest, \ + KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsList, \ SubsResponse, AlarmDescriptor, AlarmID, AlarmList, AlarmResponse, AlarmSubscription, RawKpiTable from common.proto.monitoring_pb2_grpc import MonitoringServiceStub -- GitLab From 3e7573750600b730873138e2a765a0cdabd992a7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:35:10 +0000 Subject: [PATCH 019/106] Spelling correction from "KPIMANGER" to "KPIMANAGER" --- src/common/Constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Constants.py b/src/common/Constants.py index 4a0f3a226..ee737d2bd 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -74,7 +74,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ZTP .value : 5050, ServiceNameEnum.POLICY .value : 6060, ServiceNameEnum.MONITORING .value : 7070, - ServiceNameEnum.KPIMANGER .value : 7071, + ServiceNameEnum.KPIMANAGER .value : 7071, ServiceNameEnum.DLT .value : 8080, ServiceNameEnum.NBI .value : 9090, ServiceNameEnum.L3_CAD .value : 10001, -- GitLab From 9f70a96cf4530bc074f98f4e030e6dca5e94fa57 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:35:41 +0000 Subject: [PATCH 020/106] files permission changed --- src/kpi_manager/client/KpiManagerClient.py | 0 src/kpi_manager/service/KpiManagerService.py | 0 src/kpi_manager/tests/test_messages.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/kpi_manager/client/KpiManagerClient.py mode change 100644 => 100755 src/kpi_manager/service/KpiManagerService.py mode change 100644 => 100755 src/kpi_manager/tests/test_messages.py diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py old mode 100644 new mode 100755 diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py old mode 100644 new mode 100755 diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py old mode 100644 new mode 100755 -- GitLab From 95ebbc734aeb400fc7b453878733eb5d7489a963 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:01:45 +0000 Subject: [PATCH 021/106] tests file for Kpi Manager is added --- scripts/run_tests_locally_kpi_manager.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts/run_tests_locally_kpi_manager.sh diff --git a/scripts/run_tests_locally_kpi_manager.sh b/scripts/run_tests_locally_kpi_manager.sh new file mode 100755 index 000000000..eeeec4bb2 --- /dev/null +++ b/scripts/run_tests_locally_kpi_manager.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py \ No newline at end of file -- GitLab From 50bbeb104476a21c4b981ae37ab75566d48438f4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:05:36 +0000 Subject: [PATCH 022/106] Docker build command for Kpi Manager is added and Docker file paths are updated to correct directories. --- src/build.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/build.sh b/src/build.sh index b1a7d299e..9ae91ef10 100755 --- a/src/build.sh +++ b/src/build.sh @@ -18,16 +18,20 @@ cd $(dirname $0) echo "BUILD context" -context/genproto.sh +# context/genproto.sh # genproto.sh file doesn't exist docker build -t "context:develop" -f context/Dockerfile --quiet . -docker build -t "context:test" -f context/tests/Dockerfile --quiet . +# docker build -t "context:test" -f context/tests/Dockerfile --quiet . # Dockerfile doesn't exist -cd monitoring -./genproto.sh -cd .. +# genproto.sh file doesn't exist +# cd monitoring +# ./genproto.sh +# cd .. echo "BUILD monitoring" docker build -t "monitoring:dockerfile" -f monitoring/Dockerfile . +echo "BUILD kpi manager" +docker build -t "kpi_manager:dockerfile" -f kpi_manager/Dockerfile . + echo "Prune unused images" docker image prune --force -- GitLab From e28b5ce2fcde8a28564a864c5d221d4e3e57ed63 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:07:58 +0000 Subject: [PATCH 023/106] Docker name and ports are added according to Kpi Manager Service --- src/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/start.sh b/src/start.sh index 32a016cc0..c3b5d8375 100755 --- a/src/start.sh +++ b/src/start.sh @@ -15,4 +15,4 @@ docker network create -d bridge teraflowbridge -docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge monitoring:dockerfile +docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile -- GitLab From f19ae7505ffeb37b9b4c128ac7440ebe504e8aa7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:09:28 +0000 Subject: [PATCH 024/106] monitoring docker command is added for the reference --- src/start.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/start.sh b/src/start.sh index c3b5d8375..8c3fafe6e 100755 --- a/src/start.sh +++ b/src/start.sh @@ -15,4 +15,5 @@ docker network create -d bridge teraflowbridge -docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile +# docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge monitoring:dockerfile +docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile \ No newline at end of file -- GitLab From cb1bddd6a5516fcaa1fb035ec276766f8246bf21 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:10:00 +0000 Subject: [PATCH 025/106] Kpi Manager Service docker file is created --- src/kpi_manager/Dockerfile | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/kpi_manager/Dockerfile diff --git a/src/kpi_manager/Dockerfile b/src/kpi_manager/Dockerfile new file mode 100644 index 000000000..d3d962b9f --- /dev/null +++ b/src/kpi_manager/Dockerfile @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/kpi_manager +WORKDIR /var/teraflow/kpi_manager +COPY src/kpi_manager/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ +COPY src/device/. device/ +COPY src/monitoring/. monitoring/ +COPY src/kpi_manager/. kpi_manager/ + +# Start the service +ENTRYPOINT ["python", "-m", "kpi_manager.service"] -- GitLab From 82cb396dad0a10eda54d73d70155f256b6f1f7e1 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:16:43 +0000 Subject: [PATCH 026/106] temporary requirements file is added. --- src/kpi_manager/requirements.in | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/kpi_manager/requirements.in diff --git a/src/kpi_manager/requirements.in b/src/kpi_manager/requirements.in new file mode 100644 index 000000000..a6183b57e --- /dev/null +++ b/src/kpi_manager/requirements.in @@ -0,0 +1,24 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +anytree==2.8.0 +APScheduler==3.10.1 +influx-line-protocol==0.1.4 +psycopg2-binary==2.9.3 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +questdb==1.0.1 +requests==2.27.1 +xmltodict==0.12.0 +# grpc_health_probe==0.2.0 #getting error on this library \ No newline at end of file -- GitLab From 2d868d953730ab675d629aadb74d69d297c61d89 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:17:13 +0000 Subject: [PATCH 027/106] __init__.py is added is sevice directory --- src/kpi_manager/service/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/service/__init__.py diff --git a/src/kpi_manager/service/__init__.py b/src/kpi_manager/service/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From d1d7b89af69076bad27762048422a573eab6efe0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:17:48 +0000 Subject: [PATCH 028/106] __main__.py file added in Kpi Manager service directory. --- src/kpi_manager/service/__main__.py | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/kpi_manager/service/__main__.py diff --git a/src/kpi_manager/service/__main__.py b/src/kpi_manager/service/__main__.py new file mode 100644 index 000000000..9f0e53246 --- /dev/null +++ b/src/kpi_manager/service/__main__.py @@ -0,0 +1,107 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading, time +from prometheus_client import start_http_server +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, + wait_for_environment_variables) +from common.proto import monitoring_pb2 +from monitoring.service.EventTools import EventsDeviceCollector # import updated +from monitoring.service.NameMapping import NameMapping # import updated +# from .MonitoringService import MonitoringService +from .KpiManagerService import KpiManagerService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def start_kpi_manager(name_mapping : NameMapping): + LOGGER.info('Start Monitoring...',) + + events_collector = EventsDeviceCollector(name_mapping) + events_collector.start() + + # TODO: redesign this method to be more clear and clean + + # Iterate while terminate is not set + while not terminate.is_set(): + list_new_kpi_ids = events_collector.listen_events() + + # Monitor Kpis + if bool(list_new_kpi_ids): + for kpi_id in list_new_kpi_ids: + # Create Monitor Kpi Requests + monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() + monitor_kpi_request.kpi_id.CopyFrom(kpi_id) + monitor_kpi_request.monitoring_window_s = 86400 + monitor_kpi_request.sampling_rate_s = 10 + events_collector._monitoring_client.MonitorKpi(monitor_kpi_request) + + time.sleep(0.5) # let other tasks run; do not overload CPU + else: + # Terminate is set, looping terminates + LOGGER.warning("Stopping execution...") + + events_collector.start() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + # Starting monitoring service + # grpc_service = MonitoringService(name_mapping) + # grpc_service.start() + # start_monitoring(name_mapping) + + grpc_service = KpiManagerService(name_mapping) + grpc_service.start() + + start_kpi_manager(name_mapping) + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) -- GitLab From 343483c835443c26603ca23b152294225fe0fbf3 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:19:02 +0000 Subject: [PATCH 029/106] KpiManagerService file re-structred into generic KpiManagerSevice file. --- src/kpi_manager/service/KpiManagerService.py | 106 ++----------------- 1 file changed, 10 insertions(+), 96 deletions(-) diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py index d42ce14eb..dbbcec2cf 100755 --- a/src/kpi_manager/service/KpiManagerService.py +++ b/src/kpi_manager/service/KpiManagerService.py @@ -12,106 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, os, grpc -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty - from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc +# from common.proto.monitoring_pb2_grpc import add_MonitoringServiceServicer_to_server from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server -from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer -from monitoring.service.NameMapping import NameMapping - -from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList -from monitoring.service import ManagementDBTools - from common.tools.service.GenericGrpcService import GenericGrpcService +from kpi_manager.service.KpiManagerServiceServicerImpl import KpiManagerServiceServicerImpl +# from monitoring.service.MonitoringServiceServicerImpl import MonitoringServiceServicerImpl +from monitoring.service.NameMapping import NameMapping -LOGGER = logging.getLogger(__name__) - -METRICS_POOL = MetricsPool('Monitoring', 'RPC') - -class KpiManagerServer(KpiManagerServiceServicer): - def __init__(self, cls_name: str = __name__): - LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated - GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed - - # Init sqlite monitoring db - self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? - LOGGER.info('MetricsDB initialized --- KPI Manager Service') +class KpiManagerService(GenericGrpcService): + def __init__(self, name_mapping : NameMapping, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) + super().__init__(port, cls_name=cls_name) + self.kpiManagerService_servicer = KpiManagerServiceServicerImpl(name_mapping) def install_servicers(self): - # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class - # implementation exists in the same class. - add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SetKpi( - self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: - response = KpiId() - kpi_description = request.kpi_description - kpi_sample_type = request.kpi_sample_type - kpi_device_id = request.device_id.device_uuid.uuid - kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid - kpi_service_id = request.service_id.service_uuid.uuid - kpi_slice_id = request.slice_id.slice_uuid.uuid - kpi_connection_id = request.connection_id.connection_uuid.uuid - kpi_link_id = request.link_id.link_uuid.uuid - if request.kpi_id.kpi_id.uuid != "": - response.kpi_id.uuid = request.kpi_id.kpi_id.uuid - # Here the code to modify an existing kpi - else: - data = self.management_db.insert_KPI( - kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, - kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) - response.kpi_id.uuid = str(data) - return response - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: - kpi_id = int(request.kpi_id.uuid) - kpi = self.management_db.get_KPI(kpi_id) - if kpi: - self.management_db.delete_KPI(kpi_id) - else: - LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) - return Empty() - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: - kpi_id = request.kpi_id.uuid - kpi_db = self.management_db.get_KPI(int(kpi_id)) - kpiDescriptor = KpiDescriptor() - if kpi_db is None: - LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) - else: - kpiDescriptor.kpi_description = kpi_db[1] - kpiDescriptor.kpi_sample_type = kpi_db[2] - kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) - kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) - kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) - kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) - kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) - kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) - return kpiDescriptor - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: - kpi_descriptor_list = KpiDescriptorList() - data = self.management_db.get_KPIS() - LOGGER.debug(f"data: {data}") - for item in data: - kpi_descriptor = KpiDescriptor() - kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) - kpi_descriptor.kpi_description = item[1] - kpi_descriptor.kpi_sample_type = item[2] - kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) - kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) - kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) - kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) - kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) - kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) - kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) - return kpi_descriptor_list \ No newline at end of file + add_KpiManagerServiceServicer_to_server(self.kpiManagerService_servicer, self.server) -- GitLab From 6e9b010e3a52f467f88a1c4c3181c44171295664 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:19:36 +0000 Subject: [PATCH 030/106] KpiManagerServiceImpl file is created. --- .../service/KpiManagerServiceServicerImpl.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/kpi_manager/service/KpiManagerServiceServicerImpl.py diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py new file mode 100644 index 000000000..3a40195da --- /dev/null +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -0,0 +1,104 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from monitoring.service.NameMapping import NameMapping +from monitoring.service import ManagementDBTools + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): + def __init__(self, name_mapping : NameMapping): + LOGGER.info('Init KpiManagerService') + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From ab122c221a6b64283f0afd4b21093e48e1fe54c8 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:32:21 +0000 Subject: [PATCH 031/106] These imports were added temporily --- src/kpi_manager/tests/test_unitary.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 src/kpi_manager/tests/test_unitary.py diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py old mode 100644 new mode 100755 index 39d2b2874..b75ea5751 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# import sys +# sys.path.append('.') import os, pytest import logging, json -- GitLab From 16666d8b744200e973fa30600342a9053aaada65 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:34:08 +0000 Subject: [PATCH 032/106] Imports of Kpi, KpiList and KpiDescriptor is changed to kpi_manager_pb2 file. --- src/monitoring/service/MonitoringServiceServicerImpl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/monitoring/service/MonitoringServiceServicerImpl.py b/src/monitoring/service/MonitoringServiceServicerImpl.py index 608b0bad9..e98cfa236 100644 --- a/src/monitoring/service/MonitoringServiceServicerImpl.py +++ b/src/monitoring/service/MonitoringServiceServicerImpl.py @@ -20,8 +20,8 @@ from common.proto.context_pb2 import Empty from common.proto.device_pb2 import MonitoringSettings from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.monitoring_pb2_grpc import MonitoringServiceServicer -from common.proto.monitoring_pb2 import AlarmResponse, AlarmDescriptor, AlarmList, SubsList, KpiId, \ - KpiDescriptor, KpiList, KpiQuery, SubsDescriptor, SubscriptionID, AlarmID, KpiDescriptorList, \ +from common.proto.monitoring_pb2 import AlarmResponse, AlarmDescriptor, AlarmList, SubsList, \ + KpiQuery, SubsDescriptor, SubscriptionID, AlarmID, KpiList,\ MonitorKpiRequest, Kpi, AlarmSubscription, SubsResponse, RawKpiTable, RawKpi, RawKpiList from common.tools.timestamp.Converters import timestamp_string_to_float, timestamp_utcnow_to_float from device.client.DeviceClient import DeviceClient @@ -30,6 +30,8 @@ from monitoring.service.AlarmManager import AlarmManager from monitoring.service.NameMapping import NameMapping from monitoring.service.SubscriptionManager import SubscriptionManager +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList + LOGGER = logging.getLogger(__name__) METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") -- GitLab From a3ca5899e477d26085cb239f14e5fe1d01fb8803 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:37:57 +0000 Subject: [PATCH 033/106] after temporarily removal of --rcfile file --- scripts/run_tests_locally_kpi_manager.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/run_tests_locally_kpi_manager.sh b/scripts/run_tests_locally_kpi_manager.sh index eeeec4bb2..8ed855a8e 100755 --- a/scripts/run_tests_locally_kpi_manager.sh +++ b/scripts/run_tests_locally_kpi_manager.sh @@ -17,8 +17,12 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src -RCFILE=$PROJECTDIR/coverage/.coveragerc -coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ - kpi_manager/tests/test_unitary.py +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py -# python3 kpi_manager/tests/test_unitary.py \ No newline at end of file +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --verbose \ + kpi_manager/tests/test_unitary.py \ No newline at end of file -- GitLab From 92ea40f1cd78480b7c457444b6a9861c39e935b7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:38:48 +0000 Subject: [PATCH 034/106] removal of unexpected "," --- src/kpi_manager/client/KpiManagerClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index 6baca7cef..a129cd327 100755 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -19,7 +19,7 @@ from common.Settings import get_service_host, get_service_port_grpc from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList, +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub LOGGER = logging.getLogger(__name__) -- GitLab From 7b88c37fde4204b2bf73ead2b6f0deb0ab4a7e8a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:39:29 +0000 Subject: [PATCH 035/106] __init__.py is added in kpi_manager/client directory --- src/kpi_manager/client/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/client/__init__.py diff --git a/src/kpi_manager/client/__init__.py b/src/kpi_manager/client/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From e50c6e937c4832a868dd5851586d175f8d78c638 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:40:27 +0000 Subject: [PATCH 036/106] some missing imports were added and some methods are commented temporarily --- src/kpi_manager/tests/test_unitary.py | 92 ++++++++++++++------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py index b75ea5751..113b26890 100755 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -17,6 +17,7 @@ # sys.path.append('.') import os, pytest import logging, json +from typing import Union from apscheduler.schedulers.background import BackgroundScheduler @@ -33,6 +34,8 @@ from common.tools.object_factory.Topology import json_topology, json_topology_id # from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ # AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.tools.service.GenericGrpcService import GenericGrpcService +from context.client.ContextClient import ContextClient from device.service.driver_api.DriverFactory import DriverFactory @@ -50,6 +53,9 @@ from monitoring.service.ManagementDBTools import ManagementDB from monitoring.service.MetricsDBTools import MetricsDB from monitoring.service.NameMapping import NameMapping +os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' +from device.service.drivers import DRIVERS + ########################### # Tests Setup ########################### @@ -185,49 +191,49 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab LOGGER.info('Closed KpiManagerClient...') -@pytest.fixture(scope='session') -def management_db(): - _management_db = ManagementDB('monitoring.db') - return _management_db - -@pytest.fixture(scope='session') -def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name - return monitoring_service.monitoring_servicer.metrics_db - -# This function os not clear to me (Changes should me made before execution) -@pytest.fixture(scope='session') -def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name - return monitoring_service.monitoring_servicer.metrics_db - #_metrics_db = MetricsDBTools.MetricsDB( - # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) - #return _metrics_db - -@pytest.fixture(scope='session') -def subs_scheduler(): - _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) - _scheduler.start() - return _scheduler - -def ingestion_data(kpi_id_int): - # pylint: disable=redefined-outer-name,unused-argument - metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') - - kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') - for _ in range(50): - kpiSampleType = kpiSampleType_name - kpiId = kpi_id_int - deviceId = 'DEV'+ str(kpi_id_int) - endpointId = 'END' + str(kpi_id_int) - serviceId = 'SERV' + str(kpi_id_int) - sliceId = 'SLC' + str(kpi_id_int) - connectionId = 'CON' + str(kpi_id_int) - time_stamp = timestamp_utcnow_to_float() - kpi_value = 500*random() - - metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, - kpi_value) - sleep(0.1) +# @pytest.fixture(scope='session') +# def management_db(): +# _management_db = ManagementDB('monitoring.db') +# return _management_db + +# @pytest.fixture(scope='session') +# def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name +# return monitoring_service.monitoring_servicer.metrics_db + +# # This function os not clear to me (Changes should me made before execution) +# @pytest.fixture(scope='session') +# def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name +# return monitoring_service.monitoring_servicer.metrics_db +# #_metrics_db = MetricsDBTools.MetricsDB( +# # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) +# #return _metrics_db + +# @pytest.fixture(scope='session') +# def subs_scheduler(): +# _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) +# _scheduler.start() +# return _scheduler + +# def ingestion_data(kpi_id_int): +# # pylint: disable=redefined-outer-name,unused-argument +# metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') + +# kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') +# for _ in range(50): +# kpiSampleType = kpiSampleType_name +# kpiId = kpi_id_int +# deviceId = 'DEV'+ str(kpi_id_int) +# endpointId = 'END' + str(kpi_id_int) +# serviceId = 'SERV' + str(kpi_id_int) +# sliceId = 'SLC' + str(kpi_id_int) +# connectionId = 'CON' + str(kpi_id_int) +# time_stamp = timestamp_utcnow_to_float() +# kpi_value = 500*random() + +# metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, +# kpi_value) +# sleep(0.1) ################################################## # Prepare Environment, should be the first test -- GitLab From e8f6fdba780302ed14035f43a6634053b5d1bf02 Mon Sep 17 00:00:00 2001 From: Ricard Vilalta Date: Mon, 8 Apr 2024 08:37:07 +0000 Subject: [PATCH 037/106] Upload New File --- kpi_manager.proto | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 kpi_manager.proto diff --git a/kpi_manager.proto b/kpi_manager.proto new file mode 100644 index 000000000..1ef447fb3 --- /dev/null +++ b/kpi_manager.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; +package monitoring; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service KpiManagerService { + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} + rpc SetKpiDescriptor (KpiDescriptor ) returns (KpiId ) {} + rpc DeleteKpiDescriptor(KpiId ) returns (context.Empty ) {} + rpc SelectKpiDescriptor(KpiDescriptorFilter) returns (KpiDescriptorList) {} +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + kpi_sample_types.KpiSampleType kpi_sample_type = 3; + context.DeviceId device_id = 4; + context.EndPointId endpoint_id = 5; + context.ServiceId service_id = 6; + context.SliceId slice_id = 7; + context.ConnectionId connection_id = 8; + context.LinkId link_id = 9; +} + +message KpiDescriptorFilter { + // KPI Descriptors that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all KPI Descriptors + repeated KpiId kpi_id = 1; + repeated kpi_sample_types.KpiSampleType kpi_sample_type = 2; + repeated context.DeviceId device_id = 3; + repeated context.EndPointId endpoint_id = 4; + repeated context.ServiceId service_id = 5; + repeated context.SliceId slice_id = 6; + repeated context.ConnectionId connection_id = 7; + repeated context.LinkId link_id = 8; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} -- GitLab From bbd1511c1051f60dd042303baf7f0d096ed6cc3d Mon Sep 17 00:00:00 2001 From: Ricard Vilalta Date: Mon, 8 Apr 2024 08:37:26 +0000 Subject: [PATCH 038/106] Delete kpi_manager.proto --- kpi_manager.proto | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 kpi_manager.proto diff --git a/kpi_manager.proto b/kpi_manager.proto deleted file mode 100644 index 1ef447fb3..000000000 --- a/kpi_manager.proto +++ /dev/null @@ -1,46 +0,0 @@ -syntax = "proto3"; -package monitoring; - -import "context.proto"; -import "kpi_sample_types.proto"; - -service KpiManagerService { - rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} - rpc SetKpiDescriptor (KpiDescriptor ) returns (KpiId ) {} - rpc DeleteKpiDescriptor(KpiId ) returns (context.Empty ) {} - rpc SelectKpiDescriptor(KpiDescriptorFilter) returns (KpiDescriptorList) {} -} - -message KpiId { - context.Uuid kpi_id = 1; -} - -message KpiDescriptor { - KpiId kpi_id = 1; - string kpi_description = 2; - kpi_sample_types.KpiSampleType kpi_sample_type = 3; - context.DeviceId device_id = 4; - context.EndPointId endpoint_id = 5; - context.ServiceId service_id = 6; - context.SliceId slice_id = 7; - context.ConnectionId connection_id = 8; - context.LinkId link_id = 9; -} - -message KpiDescriptorFilter { - // KPI Descriptors that fulfill the filter are those that match ALL the following fields. - // An empty list means: any value is accepted. - // All fields empty means: list all KPI Descriptors - repeated KpiId kpi_id = 1; - repeated kpi_sample_types.KpiSampleType kpi_sample_type = 2; - repeated context.DeviceId device_id = 3; - repeated context.EndPointId endpoint_id = 4; - repeated context.ServiceId service_id = 5; - repeated context.SliceId slice_id = 6; - repeated context.ConnectionId connection_id = 7; - repeated context.LinkId link_id = 8; -} - -message KpiDescriptorList { - repeated KpiDescriptor kpi_descriptor_list = 1; -} -- GitLab From b000c36edfcefa92d58d128832054559b3de1e89 Mon Sep 17 00:00:00 2001 From: Ricard Vilalta Date: Mon, 8 Apr 2024 08:41:29 +0000 Subject: [PATCH 039/106] Upload New File --- proto/telemetry_frontend.proto | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 proto/telemetry_frontend.proto diff --git a/proto/telemetry_frontend.proto b/proto/telemetry_frontend.proto new file mode 100644 index 000000000..93213628e --- /dev/null +++ b/proto/telemetry_frontend.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; +package device; + +import "context.proto"; +import "kpi_manager.proto"; +import "kpi_sample_types.proto"; + +service TelemetryFrontendService { + rpc StartCollector (Collector ) returns (CollectorId ) {} + rpc StopCollector (CollectorId ) returns (context.Empty) {} + rpc SelectCollectors(CollectorFilter) returns (CollectorList) {} +} + +message CollectorId { + context.Uuid collector_id = 1; +} + +message Collector { + CollectorId collector_id = 1; // The Collector ID + kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples + float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely + float interval_s = 4; // Interval between collected samples +} + +message CollectorFilter { + // Collector that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all Collectors + repeated CollectorId collector_id = 1; + repeated kpi_manager.KpiId kpi_id = 2; + repeated kpi_sample_types.KpiSampleType kpi_sample_type = 3; + repeated context.DeviceId device_id = 4; + repeated context.EndPointId endpoint_id = 5; + repeated context.ServiceId service_id = 6; + repeated context.SliceId slice_id = 7; + repeated context.ConnectionId connection_id = 8; + repeated context.LinkId link_id = 9; +} + +message CollectorList { + repeated Collector collector_list = 1; +} -- GitLab From 0c68bc65f7ffe72f10f95ed13ac1fc87404d5952 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 11 Apr 2024 13:50:35 +0000 Subject: [PATCH 040/106] comment added to mentioned the place of error during KPI_MANAGER tests --- src/kpi_manager/tests/test_unitary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py index 113b26890..b45346d06 100755 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -239,6 +239,7 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab # Prepare Environment, should be the first test ################################################## +# ERROR on this test --- def test_prepare_environment( context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument ): -- GitLab From 3de99d75ff905e1b38304775752d5144ec9d6143 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 11 Apr 2024 13:51:51 +0000 Subject: [PATCH 041/106] Kafka.sh is added (An automated script to deply Apache kafka) --- deploy/kafka.sh | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100755 deploy/kafka.sh diff --git a/deploy/kafka.sh b/deploy/kafka.sh new file mode 100755 index 000000000..976bec117 --- /dev/null +++ b/deploy/kafka.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Read deployment settings +######################################################################################################################## + +# If not already set, set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} + + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +# Constants +TMP_FOLDER="./tmp" +KFK_MANIFESTS_PATH="manifests/kafka" +KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" +KFK_MANIFEST="02-kafka.yaml" + +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" +mkdir -p ${TMP_MANIFESTS_FOLDER} + +# copy zookeeper and kafka manifest files to temporary manifest location +cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" +cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" + +kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found +kubectl create namespace ${KFK_NAMESPACE} +# sleep 2 +# echo "----" + +# Kafka zookeeper service should be deployed before the kafka service +kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" +# kubectl get services --namespace ${KFK_NAMESPACE} +# echo "----" + +KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to get service name automatically +KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') +# echo $KFK_ZOOKEEPER_IP +# echo "----" + +# Kafka service should be deployed after the zookeeper service +sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" +# echo "----" +kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" +sleep 5 +kubectl --namespace ${KFK_NAMESPACE} get pods + +echo "--- Kafka service deployed sucessfully ---" \ No newline at end of file -- GitLab From a0ae85f2b319f06221d7eb1f523e9ec8e052bd1f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 11 Apr 2024 13:52:31 +0000 Subject: [PATCH 042/106] Apache Kafka manifest files for deployment --- manifests/kafka/01-zookeeper.yaml | 40 +++++++++++++++++++++++++++ manifests/kafka/02-kafka.yaml | 46 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 manifests/kafka/01-zookeeper.yaml create mode 100644 manifests/kafka/02-kafka.yaml diff --git a/manifests/kafka/01-zookeeper.yaml b/manifests/kafka/01-zookeeper.yaml new file mode 100644 index 000000000..0f5ade5d9 --- /dev/null +++ b/manifests/kafka/01-zookeeper.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper-service + name: zookeeper-service + namespace: kafka +spec: + type: NodePort + ports: + - name: zookeeper-port + port: 2181 + nodePort: 30181 + targetPort: 2181 + selector: + app: zookeeper +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: zookeeper + name: zookeeper + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - image: wurstmeister/zookeeper + imagePullPolicy: IfNotPresent + name: zookeeper + ports: + - containerPort: 2181 \ No newline at end of file diff --git a/manifests/kafka/02-kafka.yaml b/manifests/kafka/02-kafka.yaml new file mode 100644 index 000000000..8a2b51724 --- /dev/null +++ b/manifests/kafka/02-kafka.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kafka-broker + name: kafka-service + namespace: kafka +spec: + ports: + - port: 9092 + selector: + app: kafka-broker +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kafka-broker + name: kafka-broker + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-broker + template: + metadata: + labels: + app: kafka-broker + spec: + hostname: kafka-broker + containers: + - env: + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: :2181 + - name: KAFKA_LISTENERS + value: PLAINTEXT://:9092 + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://localhost:9092 + image: wurstmeister/kafka + imagePullPolicy: IfNotPresent + name: kafka-broker + ports: + - containerPort: 9092 \ No newline at end of file -- GitLab From 2e0889be589f100d22ae6246ebaac0ef967010ac Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 11 Apr 2024 13:54:00 +0000 Subject: [PATCH 043/106] packages version changed to pre-tested versions. --- src/monitoring/requirements.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/monitoring/requirements.in b/src/monitoring/requirements.in index 4e57dd019..bea1bc165 100644 --- a/src/monitoring/requirements.in +++ b/src/monitoring/requirements.in @@ -32,3 +32,15 @@ requests==2.27.1 xmltodict==0.12.0 questdb==1.0.1 psycopg2-binary==2.9.3 +coverage==6.3 +grpcio==1.47.* +grpcio-health-checking==1.47.* +grpcio-tools==1.47.* +grpclib==0.4.4 +prettytable==3.5.0 +prometheus-client==0.13.0 +protobuf==3.20.* +pytest==6.2.5 +pytest-benchmark==3.4.1 +python-dateutil==2.8.2 +pytest-depends==1.0.1 -- GitLab From 2fc6a4982fea2681100e2737a8b43d22d19b6ed6 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 11 Apr 2024 13:54:46 +0000 Subject: [PATCH 044/106] Apache Kafka enviornment variable is added. --- my_deploy.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/my_deploy.sh b/my_deploy.sh index 5bd58d0dc..74c293619 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -154,3 +154,10 @@ export PROM_EXT_PORT_HTTP="9090" # Set the external port Grafana HTTP Dashboards will be exposed to. export GRAF_EXT_PORT_HTTP="3000" + + +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + -- GitLab From 3a00a55c67a4105fcadc6e9944bf8f598b6ed1b9 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 12 Apr 2024 09:05:59 +0000 Subject: [PATCH 045/106] Improvement in Kafka deployment script --- deploy/kafka.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 976bec117..f2fb666b5 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -39,26 +39,31 @@ mkdir -p ${TMP_MANIFESTS_FOLDER} cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" +echo "Apache Kafka Namespace" +echo ">>> Delete Apache Kafka Namespace" kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found + +echo ">>> Create Apache Kafka Namespace" kubectl create namespace ${KFK_NAMESPACE} -# sleep 2 -# echo "----" +echo ">>> Deplying Apache Kafka Zookeeper" # Kafka zookeeper service should be deployed before the kafka service kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" -# kubectl get services --namespace ${KFK_NAMESPACE} -# echo "----" -KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to get service name automatically +KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') -# echo $KFK_ZOOKEEPER_IP -# echo "----" # Kafka service should be deployed after the zookeeper service sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" -# echo "----" + +echo ">>> Deploying Apache Kafka Broker" kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" -sleep 5 -kubectl --namespace ${KFK_NAMESPACE} get pods -echo "--- Kafka service deployed sucessfully ---" \ No newline at end of file +echo ">>> Verifing Apache Kafka deployment" +sleep 5 +KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) +if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then + echo "Deployment Error: $KFK_PODS_STATUS" +else + echo "$KFK_PODS_STATUS" +fi \ No newline at end of file -- GitLab From 7a342bb6e3d2a70822befcf6c32f869c96cc3430 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:22:03 +0000 Subject: [PATCH 046/106] Added new message (KpiDescriptorFilter) and updated method name --- proto/kpi_manager.proto | 42 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/proto/kpi_manager.proto b/proto/kpi_manager.proto index f5769ed37..ad48eb84f 100644 --- a/proto/kpi_manager.proto +++ b/proto/kpi_manager.proto @@ -19,27 +19,41 @@ import "context.proto"; import "kpi_sample_types.proto"; service KpiManagerService{ - rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final - rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final - rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final - rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final + rpc SetKpiDescriptor (KpiDescriptor ) returns (KpiId ) {} // Stable not final + rpc DeleteKpiDescriptor (KpiId ) returns (context.Empty ) {} // Stable and final + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final + rpc SelectKpiDescriptor (KpiDescriptorFilter) returns (KpiDescriptorList ) {} // Stable and final +} + + +message KpiId { + context.Uuid kpi_id = 1; } message KpiDescriptor { KpiId kpi_id = 1; string kpi_description = 2; - repeated KpiId kpi_id_list = 3; - kpi_sample_types.KpiSampleType kpi_sample_type = 4; - context.DeviceId device_id = 5; - context.EndPointId endpoint_id = 6; - context.ServiceId service_id = 7; - context.SliceId slice_id = 8; - context.ConnectionId connection_id = 9; - context.LinkId link_id = 10; + kpi_sample_types.KpiSampleType kpi_sample_type = 3; + context.DeviceId device_id = 4; + context.EndPointId endpoint_id = 5; + context.ServiceId service_id = 6; + context.SliceId slice_id = 7; + context.ConnectionId connection_id = 8; + context.LinkId link_id = 9; } -message KpiId { - context.Uuid kpi_id = 1; +message KpiDescriptorFilter { + // KPI Descriptors that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all KPI Descriptors + repeated KpiId kpi_id = 1; + repeated kpi_sample_types.KpiSampleType kpi_sample_type = 2; + repeated context.DeviceId device_id = 3; + repeated context.EndPointId endpoint_id = 4; + repeated context.ServiceId service_id = 5; + repeated context.SliceId slice_id = 6; + repeated context.ConnectionId connection_id = 7; + repeated context.LinkId link_id = 8; } message KpiDescriptorList { -- GitLab From 2a3ddf384af7bd9751b35207588333d0bc0c06e4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:23:18 +0000 Subject: [PATCH 047/106] Update method names --- src/kpi_manager/client/KpiManagerClient.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index a129cd327..5a4cd2e20 100755 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -19,7 +19,7 @@ from common.Settings import get_service_host, get_service_port_grpc from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub LOGGER = logging.getLogger(__name__) @@ -29,8 +29,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class KpiManagerClient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) # update enum - if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # update enum + if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) self.channel = None @@ -48,17 +48,17 @@ class KpiManagerClient: self.stub = None @RETRY_DECORATOR - def SetKpi(self, request : KpiDescriptor) -> KpiId: - LOGGER.debug('SetKpi: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.SetKpi(request) - LOGGER.debug('SetKpi result: {:s}'.format(grpc_message_to_json_string(response))) + def SetKpiDescriptor(self, request : KpiDescriptor) -> KpiId: + LOGGER.debug('SetKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SetKpiDescriptor(request) + LOGGER.debug('SetKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR - def DeleteKpi(self,request : KpiId) -> Empty: - LOGGER.debug('DeleteKpi: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.DeleteKpi(request) - LOGGER.info('DeleteKpi result: {:s}'.format(grpc_message_to_json_string(response))) + def DeleteKpiDescriptor(self,request : KpiId) -> Empty: + LOGGER.debug('DeleteKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteKpiDescriptor(request) + LOGGER.info('DeleteKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR @@ -69,8 +69,8 @@ class KpiManagerClient: return response @RETRY_DECORATOR - def GetKpiDescriptorList(self, request : Empty) -> KpiDescriptorList: - LOGGER.debug('GetKpiDescriptorList: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.GetKpiDescriptorList(request) - LOGGER.debug('GetKpiDescriptorList result: {:s}'.format(grpc_message_to_json_string(response))) + def SelectKpiDescriptor(self, request : KpiDescriptorFilter) -> KpiDescriptorList: + LOGGER.debug('SelectKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SelectKpiDescriptor(request) + LOGGER.debug('SelectKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) return response \ No newline at end of file -- GitLab From bb54d7a49c083f2a7927e3259ce7a50d014498be Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:23:43 +0000 Subject: [PATCH 048/106] update method name --- src/kpi_manager/service/KpiManagerServiceServicerImpl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py index 3a40195da..fc1ea3b96 100644 --- a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -16,7 +16,7 @@ import logging, grpc from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.context_pb2 import Empty from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList from monitoring.service.NameMapping import NameMapping from monitoring.service import ManagementDBTools @@ -34,7 +34,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): LOGGER.info('MetricsDB initialized --- KPI Manager Service') @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SetKpi( + def SetKpiDescriptor( self, request: KpiDescriptor, grpc_context: grpc.ServicerContext ) -> KpiId: response = KpiId() @@ -57,7 +57,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): return response @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + def DeleteKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: kpi_id = int(request.kpi_id.uuid) kpi = self.management_db.get_KPI(kpi_id) if kpi: @@ -85,7 +85,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): return kpiDescriptor @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + def SelectKpiDescriptor(self, request: KpiDescriptorFilter, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: kpi_descriptor_list = KpiDescriptorList() data = self.management_db.get_KPIS() LOGGER.debug(f"data: {data}") -- GitLab From 4720cbc7a3b1c0beda0861839075701b488daa49 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:25:08 +0000 Subject: [PATCH 049/106] New method "create_kpi_filter_request" is added --- src/kpi_manager/tests/test_messages.py | 73 +++++++++++++++++--------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py index 589d6cb84..237a983bf 100755 --- a/src/kpi_manager/tests/test_messages.py +++ b/src/kpi_manager/tests/test_messages.py @@ -24,46 +24,67 @@ def create_kpi_request(kpi_id_str): _create_kpi_request = kpi_manager_pb2.KpiDescriptor() _create_kpi_request.kpi_description = 'KPI Description Test' _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) + _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) return _create_kpi_request def create_kpi_request_b(): - _create_kpi_request = kpi_manager_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member return _create_kpi_request def create_kpi_request_c(): - _create_kpi_request = kpi_manager_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member return _create_kpi_request def create_kpi_request_d(): - _create_kpi_request = kpi_manager_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member return _create_kpi_request def kpi_descriptor_list(): _kpi_descriptor_list = kpi_manager_pb2.KpiDescriptorList() - return _kpi_descriptor_list \ No newline at end of file + return _kpi_descriptor_list + +def create_kpi_filter_request(): + _create_kpi_filter_request = kpi_manager_pb2.KpiDescriptorFilter() + _create_kpi_filter_request.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) + new_device_id = _create_kpi_filter_request.device_id.add() + new_device_id.device_uuid.uuid = 'DEV1' + new_service_id = _create_kpi_filter_request.service_id.add() + new_service_id.service_uuid.uuid = 'SERV1' + new_slice_id = _create_kpi_filter_request.slice_id.add() + new_slice_id.slice_uuid.uuid = 'SLC1' + new_endpoint_id = _create_kpi_filter_request.endpoint_id.add() + new_endpoint_id.endpoint_uuid.uuid = 'END1' + new_connection_id = _create_kpi_filter_request.connection_id.add() + new_connection_id.connection_uuid.uuid = 'CON1' + + # _create_kpi_filter_request.device_id.device_uuid.uuid.append('DEV1') # pylint: disable=maybe-no-member + # _create_kpi_filter_request[0].service_id.service_uuid.uuid = 'SERV1' # pylint: disable=maybe-no-member + # _create_kpi_filter_request[0].slice_id.slice_uuid.uuid = 'SLC1' # pylint: disable=maybe-no-member + # _create_kpi_filter_request[0].endpoint_id.endpoint_uuid.uuid = 'END1' # pylint: disable=maybe-no-member + # _create_kpi_filter_request[0].connection_id.connection_uuid.uuid = 'CON1' # pylint: disable=maybe-no-member + return _create_kpi_filter_request -- GitLab From 44158d341973232e47826cde8a7b50406dfab10f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:25:29 +0000 Subject: [PATCH 050/106] comments were removed --- src/kpi_manager/tests/test_messages.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py index 237a983bf..72ff74c16 100755 --- a/src/kpi_manager/tests/test_messages.py +++ b/src/kpi_manager/tests/test_messages.py @@ -82,9 +82,4 @@ def create_kpi_filter_request(): new_connection_id = _create_kpi_filter_request.connection_id.add() new_connection_id.connection_uuid.uuid = 'CON1' - # _create_kpi_filter_request.device_id.device_uuid.uuid.append('DEV1') # pylint: disable=maybe-no-member - # _create_kpi_filter_request[0].service_id.service_uuid.uuid = 'SERV1' # pylint: disable=maybe-no-member - # _create_kpi_filter_request[0].slice_id.slice_uuid.uuid = 'SLC1' # pylint: disable=maybe-no-member - # _create_kpi_filter_request[0].endpoint_id.endpoint_uuid.uuid = 'END1' # pylint: disable=maybe-no-member - # _create_kpi_filter_request[0].connection_id.connection_uuid.uuid = 'CON1' # pylint: disable=maybe-no-member - return _create_kpi_filter_request + return _create_kpi_filter_request \ No newline at end of file -- GitLab From 843c37fbf969fcc2261b72a1171ad1c0bfd675e0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 16 Apr 2024 09:26:39 +0000 Subject: [PATCH 051/106] function calls were updated according to name method names --- src/kpi_manager/tests/test_unitary.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py index b45346d06..adc886341 100755 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -33,7 +33,7 @@ from common.tools.object_factory.Context import json_context, json_context_id from common.tools.object_factory.Topology import json_topology, json_topology_id # from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ # AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList from common.tools.service.GenericGrpcService import GenericGrpcService from context.client.ContextClient import ContextClient @@ -43,7 +43,7 @@ from device.service.driver_api.DriverInstanceCache import DriverInstanceCache from device.service.DeviceService import DeviceService from device.client.DeviceClient import DeviceClient -from kpi_manager.tests.test_messages import create_kpi_request, create_kpi_request_b, create_kpi_request_c, create_kpi_request_d +from kpi_manager.tests.test_messages import create_kpi_request, create_kpi_request_b, create_kpi_request_c, create_kpi_request_d, create_kpi_filter_request # from monitoring.service.MonitoringService import MonitoringService from kpi_manager.service.KpiManagerService import KpiManagerService # from monitoring.client.MonitoringClient import MonitoringClient @@ -256,7 +256,7 @@ def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name # make call to server LOGGER.warning('test_create_kpi requesting') for i in range(3): - response = kpi_manager_client.SetKpi(create_kpi_request(str(i+1))) + response = kpi_manager_client.SetKpiDescriptor(create_kpi_request(str(i+1))) LOGGER.debug(str(response)) assert isinstance(response, KpiId) @@ -264,14 +264,14 @@ def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name def test_delete_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name # make call to server LOGGER.warning('delete_kpi requesting') - response = kpi_manager_client.SetKpi(create_kpi_request('4')) - response = kpi_manager_client.DeleteKpi(response) + response = kpi_manager_client.SetKpiDescriptor(create_kpi_request('4')) + response = kpi_manager_client.DeleteKpiDescriptor(response) LOGGER.debug(str(response)) assert isinstance(response, Empty) # Test case that makes use of client fixture to test server's GetKpiDescriptor method -def test_get_kpi_descriptor_list(kpi_manager_client): # pylint: disable=redefined-outer-name - LOGGER.warning('test_getkpidescritor_kpi begin') - response = kpi_manager_client.GetKpiDescriptorList(Empty()) +def test_select_kpi_descriptor(kpi_manager_client): # pylint: disable=redefined-outer-name + LOGGER.warning('test_selectkpidescritor begin') + response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) LOGGER.debug(str(response)) assert isinstance(response, KpiDescriptorList) -- GitLab From acf03e1207b1a5c7f733f31de31846bc77053b19 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:19:36 +0000 Subject: [PATCH 052/106] Improvements in file formatting --- proto/telemetry_frontend.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/telemetry_frontend.proto b/proto/telemetry_frontend.proto index 93213628e..1f89a5d54 100644 --- a/proto/telemetry_frontend.proto +++ b/proto/telemetry_frontend.proto @@ -16,10 +16,10 @@ message CollectorId { } message Collector { - CollectorId collector_id = 1; // The Collector ID - kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples - float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely - float interval_s = 4; // Interval between collected samples + CollectorId collector_id = 1; // The Collector ID + kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples + float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely + float interval_s = 4; // Interval between collected samples } message CollectorFilter { -- GitLab From 86c1d4eacadd4301bec08cdcf28044ddedeb95b4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:20:25 +0000 Subject: [PATCH 053/106] "Telemetry Frontend" Enum name and port added --- src/common/Constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/Constants.py b/src/common/Constants.py index ee737d2bd..72003846b 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -44,6 +44,7 @@ class ServiceNameEnum(Enum): POLICY = 'policy' MONITORING = 'monitoring' KPIMANAGER = 'kpiManager' + TELEMETRYFRONTEND = 'telemetryfrontend' DLT = 'dlt' NBI = 'nbi' CYBERSECURITY = 'cybersecurity' @@ -75,6 +76,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.POLICY .value : 6060, ServiceNameEnum.MONITORING .value : 7070, ServiceNameEnum.KPIMANAGER .value : 7071, + ServiceNameEnum.TELEMETRYFRONTEND .value : 7072, ServiceNameEnum.DLT .value : 8080, ServiceNameEnum.NBI .value : 9090, ServiceNameEnum.L3_CAD .value : 10001, -- GitLab From f581f2e3ec830b4319ccd8378eaf6dcb88ab15cb Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:21:51 +0000 Subject: [PATCH 054/106] improvements in file formatting and MAX_RETRIES changed to 10 from 15. --- src/kpi_manager/client/KpiManagerClient.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index 5a4cd2e20..30b1720fb 100755 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -16,14 +16,14 @@ import grpc, logging from common.Constants import ServiceNameEnum from common.Settings import get_service_host, get_service_port_grpc -from common.tools.client.RetryDecorator import retry, delay_exponential -from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList LOGGER = logging.getLogger(__name__) -MAX_RETRIES = 15 +MAX_RETRIES = 10 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') -- GitLab From 292df45380d0b9a6baeb2166e5464ee32f0f2037 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:22:46 +0000 Subject: [PATCH 055/106] Initial "Telemetry Frontend" client code --- .../client/TelemetryFrontendClient.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/telemetry_frontend/client/TelemetryFrontendClient.py diff --git a/src/telemetry_frontend/client/TelemetryFrontendClient.py b/src/telemetry_frontend/client/TelemetryFrontendClient.py new file mode 100644 index 000000000..9ca19bb8c --- /dev/null +++ b/src/telemetry_frontend/client/TelemetryFrontendClient.py @@ -0,0 +1,70 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceStub +from comment.proto.telemetry_frontend_pb2 import Collector, CollectorId, CollectorFilter, CollectorList + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class TelemetryFrontendClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.TELEMETRYFRONTEND) + if not port: port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = TelemetryFrontendServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def StartCollector(self, request : Collector) --> CollectorId: + LOGGER.debug('StartCollector: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StartCollector(request) + LOGGER.debug('StartCollector result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def StopCollector(self, request : CollectorId) --> Empty: + LOGGER.debug('StopCollector: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StopCollector(request) + LOGGER.debug('StopCollector result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SelectCollectors(self, request : CollectorFilter) --> CollectorList: + LOGGER.debug('SelectCollectors: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SelectCollectors(request) + LOGGER.debug('SelectCollectors result: {:s}'.format(grpc_message_to_json_string(response))) + return response + -- GitLab From 55deaf8425de3a736a5fbf574295a2477f9a793e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:24:05 +0000 Subject: [PATCH 056/106] __init__.py added in "Telemetry Frontend" subdirectories. --- src/telemetry_frontend/__init__.py | 1 + src/telemetry_frontend/client/__init__.py | 14 ++++++++++++++ src/telemetry_frontend/service/__init__.py | 0 3 files changed, 15 insertions(+) create mode 100644 src/telemetry_frontend/__init__.py create mode 100644 src/telemetry_frontend/client/__init__.py create mode 100644 src/telemetry_frontend/service/__init__.py diff --git a/src/telemetry_frontend/__init__.py b/src/telemetry_frontend/__init__.py new file mode 100644 index 000000000..eb1ae458f --- /dev/null +++ b/src/telemetry_frontend/__init__.py @@ -0,0 +1 @@ +... diff --git a/src/telemetry_frontend/client/__init__.py b/src/telemetry_frontend/client/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/telemetry_frontend/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry_frontend/service/__init__.py b/src/telemetry_frontend/service/__init__.py new file mode 100644 index 000000000..e69de29bb -- GitLab From b7df322b4cc2b728168474cf21924f72cbfd2372 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:50:22 +0000 Subject: [PATCH 057/106] "TelemetryFrontendServiceServicerImpl" file added --- .../service/TelemetryFrontendServiceServicerImpl.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py new file mode 100644 index 000000000..f80ccfd52 --- /dev/null +++ b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file -- GitLab From a39e8f074701d1d79af7fc71e593ca71da39d6d2 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 24 Apr 2024 16:51:01 +0000 Subject: [PATCH 058/106] Initial "TelemetryFrontendService" generic file added --- .../service/TelemetryFrontendService.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/telemetry_frontend/service/TelemetryFrontendService.py diff --git a/src/telemetry_frontend/service/TelemetryFrontendService.py b/src/telemetry_frontend/service/TelemetryFrontendService.py new file mode 100644 index 000000000..16e77e61b --- /dev/null +++ b/src/telemetry_frontend/service/TelemetryFrontendService.py @@ -0,0 +1,30 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from monitoring.service.NameMapping import NameMapping +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.proto.telemetry_frontend_pb2_grpc import add_TelemetryFrontendServiceServicer_to_server +from telemetryfrontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl + + +class TelemetryFrontendService(GenericGrpcService): + def __init__(self, name_mapping : NameMapping, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + super().__init__(port, cls_name=cls_name) + self.telemetry_frontend_servicer = TelemetryFrontendServiceServicerImpl(name_mapping) + + def install_servicers(self): + add_TelemetryFrontendServiceServicer_to_server(self.telemetry_frontend_servicer, self.server) -- GitLab From faeed6d72cd393b7a6643a9038651d42b1395e94 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 10:12:58 +0000 Subject: [PATCH 059/106] Metric pool sub-module name changed from "RPC" to "KpiManager" --- src/kpi_manager/service/KpiManagerServiceServicerImpl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py index fc1ea3b96..4ffa1d2a6 100644 --- a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# do tests to verify the "grpc.ServicerContext" is required or not. import logging, grpc from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.context_pb2 import Empty @@ -23,7 +24,7 @@ from monitoring.service import ManagementDBTools LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('Monitoring', 'RPC') +METRICS_POOL = MetricsPool('Monitoring', 'KpiManager') class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): def __init__(self, name_mapping : NameMapping): @@ -36,7 +37,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SetKpiDescriptor( self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: + ) -> KpiId: response = KpiId() kpi_description = request.kpi_description kpi_sample_type = request.kpi_sample_type -- GitLab From c8b75826bcda70d652b9c3c2a534b33ada39f8c7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 10:13:44 +0000 Subject: [PATCH 060/106] "__init__.py" file added in TelemeteryFrontend sub-directories --- src/telemetry_frontend/service/__init__.py | 14 ++++++++++++++ src/telemetry_frontend/tests/__init__.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 src/telemetry_frontend/tests/__init__.py diff --git a/src/telemetry_frontend/service/__init__.py b/src/telemetry_frontend/service/__init__.py index e69de29bb..1549d9811 100644 --- a/src/telemetry_frontend/service/__init__.py +++ b/src/telemetry_frontend/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry_frontend/tests/__init__.py b/src/telemetry_frontend/tests/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/telemetry_frontend/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From dd0f90fad5e20056bf2e2f2b799464b88391ce84 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 10:14:37 +0000 Subject: [PATCH 061/106] "StartCollector" method dummy defination added --- .../TelemetryFrontendServiceServicerImpl.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py index f80ccfd52..6ffb78ab1 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -10,4 +10,29 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +import logging +from monitoring.service.NameMapping import NameMapping +from common.proto.telemetry_frontend_pb2 import CollectorId, Collector +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceServicer + + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('Monitoring', 'TelemetryFrontend') + +class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): + def __init__(self, name_mapping : NameMapping): + LOGGER.info('Init TelemetryFrontendService') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StartCollector(self, request: Collector) -> CollectorId: + response = CollectorId() + collector_id = request.collector_id + collector_kpi_id = request.kpi_id + collector_duration = request.duration_s + collector_interval = request.interval_s + + response.collector_id.uuid = request.collector_id.uuid + return response \ No newline at end of file -- GitLab From d66e79c3cbaf8a70e064689aae2690be658ccf29 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 10:15:29 +0000 Subject: [PATCH 062/106] method to test "collectorId" message is added --- src/telemetry_frontend/tests/test_mesages.py | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/telemetry_frontend/tests/test_mesages.py diff --git a/src/telemetry_frontend/tests/test_mesages.py b/src/telemetry_frontend/tests/test_mesages.py new file mode 100644 index 000000000..0b7eaeb2b --- /dev/null +++ b/src/telemetry_frontend/tests/test_mesages.py @@ -0,0 +1,22 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.proto import telemetry_frontend_pb2 +# from common.proto.kpi_manager_pb2 import kpi_id +# from common.proto.kpi_sample_types_pb2 import KpiSampleType + +def collector_id(): + _collector_id = telemetry_frontend_pb2.CollectorId() + _collector_id.collector_id.uuid = str(1) + return _collector_id \ No newline at end of file -- GitLab From e03a7344575d87aeaa234ada771d15205552e6b1 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 10:16:27 +0000 Subject: [PATCH 063/106] __main__.py function is added in TelemetryFrontend directory --- src/telemetry_frontend/service/__main__.py | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/telemetry_frontend/service/__main__.py diff --git a/src/telemetry_frontend/service/__main__.py b/src/telemetry_frontend/service/__main__.py new file mode 100644 index 000000000..9b5fe70de --- /dev/null +++ b/src/telemetry_frontend/service/__main__.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, threading +from .TelemetryFrontendService import TelemetryFrontendService +from monitoring.service.NameMapping import NameMapping +from monitoring.service.EventTools import EventsDeviceCollector +from common.Settings import ( + get_log_level, wait_for_environment_variables, get_env_var_name, + get_metrics_port ) + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER + + log_level = get_log_level() + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + LOGGER = logging.getLogger(__name__) + +# ------- will be added later -------------- + # wait_for_environment_variables([ + # get_env_var_name + + + # ]) +# ------- will be added later -------------- + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + + grpc_service = TelemetryFrontendService(name_mapping) + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file -- GitLab From af3678e5618f7c3f038da5b307bbc1afe17e8781 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 15:08:50 +0000 Subject: [PATCH 064/106] test script for telemetry frontend --- .../run_tests_locally_telemetry-frontend.sh | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 scripts/run_tests_locally_telemetry-frontend.sh diff --git a/scripts/run_tests_locally_telemetry-frontend.sh b/scripts/run_tests_locally_telemetry-frontend.sh new file mode 100755 index 000000000..ac59f6dde --- /dev/null +++ b/scripts/run_tests_locally_telemetry-frontend.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --verbose \ + telemetry_frontend/tests/test_unitary.py \ No newline at end of file -- GitLab From 48afe65f0b4122f768f045c3abb5d312727f0180 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 25 Apr 2024 15:09:35 +0000 Subject: [PATCH 065/106] added __init_.py file --- src/kpi_manager/tests/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/tests/__init__.py diff --git a/src/kpi_manager/tests/__init__.py b/src/kpi_manager/tests/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From 2f8a2a3371ba4f638a0f03bb41f7b4949a8626a4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:05:34 +0000 Subject: [PATCH 066/106] improvements in comments --- src/kpi_manager/tests/test_unitary.py | 46 +-------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py index adc886341..75987a5f4 100755 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -191,50 +191,6 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab LOGGER.info('Closed KpiManagerClient...') -# @pytest.fixture(scope='session') -# def management_db(): -# _management_db = ManagementDB('monitoring.db') -# return _management_db - -# @pytest.fixture(scope='session') -# def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name -# return monitoring_service.monitoring_servicer.metrics_db - -# # This function os not clear to me (Changes should me made before execution) -# @pytest.fixture(scope='session') -# def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name -# return monitoring_service.monitoring_servicer.metrics_db -# #_metrics_db = MetricsDBTools.MetricsDB( -# # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) -# #return _metrics_db - -# @pytest.fixture(scope='session') -# def subs_scheduler(): -# _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) -# _scheduler.start() -# return _scheduler - -# def ingestion_data(kpi_id_int): -# # pylint: disable=redefined-outer-name,unused-argument -# metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') - -# kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED -# kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') -# for _ in range(50): -# kpiSampleType = kpiSampleType_name -# kpiId = kpi_id_int -# deviceId = 'DEV'+ str(kpi_id_int) -# endpointId = 'END' + str(kpi_id_int) -# serviceId = 'SERV' + str(kpi_id_int) -# sliceId = 'SLC' + str(kpi_id_int) -# connectionId = 'CON' + str(kpi_id_int) -# time_stamp = timestamp_utcnow_to_float() -# kpi_value = 500*random() - -# metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, -# kpi_value) -# sleep(0.1) - ################################################## # Prepare Environment, should be the first test ################################################## @@ -248,7 +204,7 @@ def test_prepare_environment( context_client.SetTopology(Topology(**json_topology(DEFAULT_TOPOLOGY_NAME, context_id=context_id))) ########################### -# Tests Implementation +# Tests Implementation of Kpi Manager ########################### # Test case that makes use of client fixture to test server's CreateKpi method -- GitLab From 8cf8288c3ccf6d81d3f415fd50c0517b19982f28 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:06:59 +0000 Subject: [PATCH 067/106] few syntax errors are removed --- src/telemetry_frontend/client/TelemetryFrontendClient.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/telemetry_frontend/client/TelemetryFrontendClient.py b/src/telemetry_frontend/client/TelemetryFrontendClient.py index 9ca19bb8c..a215dd408 100644 --- a/src/telemetry_frontend/client/TelemetryFrontendClient.py +++ b/src/telemetry_frontend/client/TelemetryFrontendClient.py @@ -20,7 +20,7 @@ from common.proto.context_pb2 import Empty from common.tools.grpc.Tools import grpc_message_to_json_string from common.tools.client.RetryDecorator import retry, delay_exponential from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceStub -from comment.proto.telemetry_frontend_pb2 import Collector, CollectorId, CollectorFilter, CollectorList +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId, CollectorFilter, CollectorList LOGGER = logging.getLogger(__name__) MAX_RETRIES = 10 @@ -48,21 +48,21 @@ class TelemetryFrontendClient: self.stub = None @RETRY_DECORATOR - def StartCollector(self, request : Collector) --> CollectorId: + def StartCollector(self, request : Collector) -> CollectorId: LOGGER.debug('StartCollector: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.StartCollector(request) LOGGER.debug('StartCollector result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR - def StopCollector(self, request : CollectorId) --> Empty: + def StopCollector(self, request : CollectorId) -> Empty: LOGGER.debug('StopCollector: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.StopCollector(request) LOGGER.debug('StopCollector result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR - def SelectCollectors(self, request : CollectorFilter) --> CollectorList: + def SelectCollectors(self, request : CollectorFilter) -> CollectorList: LOGGER.debug('SelectCollectors: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.SelectCollectors(request) LOGGER.debug('SelectCollectors result: {:s}'.format(grpc_message_to_json_string(response))) -- GitLab From 70ebba4213af67e7cd762631c7700f7df4c681e1 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:08:01 +0000 Subject: [PATCH 068/106] corrected package name to "telemetry_frontend" --- src/telemetry_frontend/service/TelemetryFrontendService.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry_frontend/service/TelemetryFrontendService.py b/src/telemetry_frontend/service/TelemetryFrontendService.py index 16e77e61b..a0ae704d3 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendService.py +++ b/src/telemetry_frontend/service/TelemetryFrontendService.py @@ -17,7 +17,7 @@ from common.Settings import get_service_port_grpc from monitoring.service.NameMapping import NameMapping from common.tools.service.GenericGrpcService import GenericGrpcService from common.proto.telemetry_frontend_pb2_grpc import add_TelemetryFrontendServiceServicer_to_server -from telemetryfrontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl +from telemetry_frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl class TelemetryFrontendService(GenericGrpcService): -- GitLab From e921ccd330ac0720591cc3d1417ec719944d80b2 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:08:50 +0000 Subject: [PATCH 069/106] New messages test file added in "telemetry_frontend" --- src/telemetry_frontend/tests/Messages.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/telemetry_frontend/tests/Messages.py diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry_frontend/tests/Messages.py new file mode 100644 index 000000000..86b454004 --- /dev/null +++ b/src/telemetry_frontend/tests/Messages.py @@ -0,0 +1,30 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.proto import telemetry_frontend_pb2 +# from common.proto.kpi_manager_pb2 import kpi_id +# from common.proto.kpi_sample_types_pb2 import KpiSampleType + +def collector_id(): + _collector_id = telemetry_frontend_pb2.CollectorId() + _collector_id.collector_id.uuid = str(1) + return _collector_id + +def create_collector_request(coll_id_str): + _create_collector_request = telemetry_frontend_pb2.Collector() + _create_collector_request.collector_id.uuid = str(coll_id_str) + _create_collector_request.kpi_id.kpi_uuid.uuid = 'KPIid' + str(coll_id_str) + _create_collector_request.duration_s = float(-1) + _create_collector_request.interval_s = float(-1) + return _create_collector_request -- GitLab From c73dc9a5eb79edfd53f826653e0a5baeacc2f7af Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:10:40 +0000 Subject: [PATCH 070/106] "test_messages.py" renamed to "Messages.py" --- src/telemetry_frontend/tests/test_mesages.py | 22 -------------------- 1 file changed, 22 deletions(-) delete mode 100644 src/telemetry_frontend/tests/test_mesages.py diff --git a/src/telemetry_frontend/tests/test_mesages.py b/src/telemetry_frontend/tests/test_mesages.py deleted file mode 100644 index 0b7eaeb2b..000000000 --- a/src/telemetry_frontend/tests/test_mesages.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from common.proto import telemetry_frontend_pb2 -# from common.proto.kpi_manager_pb2 import kpi_id -# from common.proto.kpi_sample_types_pb2 import KpiSampleType - -def collector_id(): - _collector_id = telemetry_frontend_pb2.CollectorId() - _collector_id.collector_id.uuid = str(1) - return _collector_id \ No newline at end of file -- GitLab From e994a4a1a29765f4142795fc01225350f263b385 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:24:27 +0000 Subject: [PATCH 071/106] "test_unitary.py" file is added in "telemetery_frontend" --- src/telemetry_frontend/tests/test_unitary.py | 166 +++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 src/telemetry_frontend/tests/test_unitary.py diff --git a/src/telemetry_frontend/tests/test_unitary.py b/src/telemetry_frontend/tests/test_unitary.py new file mode 100644 index 000000000..c52d68921 --- /dev/null +++ b/src/telemetry_frontend/tests/test_unitary.py @@ -0,0 +1,166 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +from common.Constants import ServiceNameEnum +from common.proto.telemetry_frontend_pb2 import CollectorId +from context.client.ContextClient import ContextClient + + +from telemetry_frontend.client.TelemetryFrontendClient import TelemetryFrontendClient +from telemetry_frontend.service.TelemetryFrontendService import TelemetryFrontendService +from telemetry_frontend.tests.Messages import create_collector_request + +from common.Settings import ( + get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) + +from device.client.DeviceClient import DeviceClient +from device.service.DeviceService import DeviceService +from device.service.driver_api.DriverFactory import DriverFactory +from device.service.driver_api.DriverInstanceCache import DriverInstanceCache + +from monitoring.service.NameMapping import NameMapping + +os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' +from device.service.drivers import DRIVERS + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 + +TELEMETRY_FRONTEND_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(TELEMETRY_FRONTEND_PORT) + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of Monitoring + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +@pytest.fixture(scope='session') +def context_service(): + LOGGER.info('Initializing MockContextService...') + _service = MockContextService(MOCKSERVICE_PORT) + _service.start() + + LOGGER.info('Yielding MockContextService...') + yield _service + + LOGGER.info('Terminating MockContextService...') + _service.context_servicer.msg_broker.terminate() + _service.stop() + + LOGGER.info('Terminated MockContextService...') + +@pytest.fixture(scope='session') +def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing ContextClient...') + _client = ContextClient() + + LOGGER.info('Yielding ContextClient...') + yield _client + + LOGGER.info('Closing ContextClient...') + _client.close() + + LOGGER.info('Closed ContextClient...') + +@pytest.fixture(scope='session') +def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceService...') + driver_factory = DriverFactory(DRIVERS) + driver_instance_cache = DriverInstanceCache(driver_factory) + _service = DeviceService(driver_instance_cache) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding DeviceService...') + yield _service + + LOGGER.info('Terminating DeviceService...') + _service.stop() + + LOGGER.info('Terminated DeviceService...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +@pytest.fixture(scope='session') +def telemetryFrontend_service( + context_service : MockContextService, + device_service : DeviceService + ): + LOGGER.info('Initializing TelemetryFrontendService...') + name_mapping = NameMapping() + + _service = TelemetryFrontendService(name_mapping) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding TelemetryFrontendService...') + yield _service + + LOGGER.info('Terminating TelemetryFrontendService...') + _service.stop() + + LOGGER.info('Terminated TelemetryFrontendService...') + +@pytest.fixture(scope='session') +def telemetryFrontend_client( + telemetryFrontend_service : TelemetryFrontendService + ): + LOGGER.info('Initializing TelemetryFrontendClient...') + _client = TelemetryFrontendClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding TelemetryFrontendClient...') + yield _client + + LOGGER.info('Closing TelemetryFrontendClient...') + _client.close() + + LOGGER.info('Closed TelemetryFrontendClient...') + + +########################### +# Tests Implementation of Telemetry Frontend +########################### +def test_start_collector(telemetryFrontend_client): + LOGGER.warning('test_start_collector requesting') + response = telemetryFrontend_client.StartCollector(create_collector_request('1')) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorId) + -- GitLab From 2f8cc53e86e0502aedc78eda071159f14ec99252 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:25:24 +0000 Subject: [PATCH 072/106] "# type: ignore" is added to surpass the warning message --- src/telemetry_frontend/client/TelemetryFrontendClient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/telemetry_frontend/client/TelemetryFrontendClient.py b/src/telemetry_frontend/client/TelemetryFrontendClient.py index a215dd408..9b4e27b36 100644 --- a/src/telemetry_frontend/client/TelemetryFrontendClient.py +++ b/src/telemetry_frontend/client/TelemetryFrontendClient.py @@ -48,21 +48,21 @@ class TelemetryFrontendClient: self.stub = None @RETRY_DECORATOR - def StartCollector(self, request : Collector) -> CollectorId: + def StartCollector(self, request : Collector) -> CollectorId: # type: ignore LOGGER.debug('StartCollector: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.StartCollector(request) LOGGER.debug('StartCollector result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR - def StopCollector(self, request : CollectorId) -> Empty: + def StopCollector(self, request : CollectorId) -> Empty: # type: ignore LOGGER.debug('StopCollector: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.StopCollector(request) LOGGER.debug('StopCollector result: {:s}'.format(grpc_message_to_json_string(response))) return response @RETRY_DECORATOR - def SelectCollectors(self, request : CollectorFilter) -> CollectorList: + def SelectCollectors(self, request : CollectorFilter) -> CollectorList: # type: ignore LOGGER.debug('SelectCollectors: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.SelectCollectors(request) LOGGER.debug('SelectCollectors result: {:s}'.format(grpc_message_to_json_string(response))) -- GitLab From 79423dca2735fe4d06b6a6c64620f7250b388a7f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 07:26:36 +0000 Subject: [PATCH 073/106] New imports were corrected/added --- src/telemetry_frontend/tests/test_unitary.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/telemetry_frontend/tests/test_unitary.py b/src/telemetry_frontend/tests/test_unitary.py index c52d68921..4f37514dc 100644 --- a/src/telemetry_frontend/tests/test_unitary.py +++ b/src/telemetry_frontend/tests/test_unitary.py @@ -13,19 +13,22 @@ # limitations under the License. import os +import pytest import logging +from typing import Union from common.Constants import ServiceNameEnum from common.proto.telemetry_frontend_pb2 import CollectorId +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server from context.client.ContextClient import ContextClient - +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.Settings import ( + get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) from telemetry_frontend.client.TelemetryFrontendClient import TelemetryFrontendClient from telemetry_frontend.service.TelemetryFrontendService import TelemetryFrontendService from telemetry_frontend.tests.Messages import create_collector_request -from common.Settings import ( - get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) - from device.client.DeviceClient import DeviceClient from device.service.DeviceService import DeviceService from device.service.driver_api.DriverFactory import DriverFactory -- GitLab From 8a0adf79a66595af009a5d808141d4df13e6f118 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 10:29:33 +0000 Subject: [PATCH 074/106] Added specific requirements in "telemetry_frontend" --- src/telemetry_frontend/requirements.in | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/telemetry_frontend/requirements.in diff --git a/src/telemetry_frontend/requirements.in b/src/telemetry_frontend/requirements.in new file mode 100644 index 000000000..1dd24fe32 --- /dev/null +++ b/src/telemetry_frontend/requirements.in @@ -0,0 +1,24 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +anytree==2.8.0 +APScheduler==3.10.1 +influx-line-protocol==0.1.4 +psycopg2-binary==2.9.3 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +pytz==2024.1 +questdb==1.0.1 +requests==2.27.1 +xmltodict==0.12.0 \ No newline at end of file -- GitLab From 7c705a70add69fde12435984a53d7911c8c12492 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 10:30:49 +0000 Subject: [PATCH 075/106] Add "# type: ignore" to surpass warning message --- .../service/KpiManagerServiceServicerImpl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py index 4ffa1d2a6..f1d370f30 100644 --- a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -36,8 +36,8 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SetKpiDescriptor( - self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext # type: ignore + ) -> KpiId: # type: ignore response = KpiId() kpi_description = request.kpi_description kpi_sample_type = request.kpi_sample_type @@ -58,7 +58,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): return response @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + def DeleteKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: # type: ignore kpi_id = int(request.kpi_id.uuid) kpi = self.management_db.get_KPI(kpi_id) if kpi: @@ -68,7 +68,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): return Empty() @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: # type: ignore kpi_id = request.kpi_id.uuid kpi_db = self.management_db.get_KPI(int(kpi_id)) kpiDescriptor = KpiDescriptor() @@ -86,7 +86,7 @@ class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): return kpiDescriptor @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SelectKpiDescriptor(self, request: KpiDescriptorFilter, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + def SelectKpiDescriptor(self, request: KpiDescriptorFilter, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: # type: ignore kpi_descriptor_list = KpiDescriptorList() data = self.management_db.get_KPIS() LOGGER.debug(f"data: {data}") -- GitLab From 21f0b22aca943ee14cf9d6e20ed8e32bafcd3696 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 10:31:35 +0000 Subject: [PATCH 076/106] Adds dummy logic in "StartCollector" function --- .../TelemetryFrontendServiceServicerImpl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py index 6ffb78ab1..7814107dd 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import grpc import logging from monitoring.service.NameMapping import NameMapping from common.proto.telemetry_frontend_pb2 import CollectorId, Collector @@ -27,12 +28,13 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): LOGGER.info('Init TelemetryFrontendService') @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def StartCollector(self, request: Collector) -> CollectorId: + def StartCollector(self, request : Collector, grpc_context: grpc.ServicerContext # type: ignore + ) -> CollectorId: # type: ignore response = CollectorId() - collector_id = request.collector_id - collector_kpi_id = request.kpi_id - collector_duration = request.duration_s - collector_interval = request.interval_s + _collector_id = request.collector_id + # collector_kpi_id = request.kpi_id + # collector_duration = request.duration_s + # collector_interval = request.interval_s - response.collector_id.uuid = request.collector_id.uuid + response.collector_id.uuid = _collector_id.collector_id.uuid return response \ No newline at end of file -- GitLab From 29b6cfbd1009560aa808280d35f22555889c4b4c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 10:33:38 +0000 Subject: [PATCH 077/106] Correction of "collector_id.uuid" and "kpi_id.uuid" assignments --- src/telemetry_frontend/tests/Messages.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry_frontend/tests/Messages.py index 86b454004..d63422b72 100644 --- a/src/telemetry_frontend/tests/Messages.py +++ b/src/telemetry_frontend/tests/Messages.py @@ -22,9 +22,9 @@ def collector_id(): return _collector_id def create_collector_request(coll_id_str): - _create_collector_request = telemetry_frontend_pb2.Collector() - _create_collector_request.collector_id.uuid = str(coll_id_str) - _create_collector_request.kpi_id.kpi_uuid.uuid = 'KPIid' + str(coll_id_str) - _create_collector_request.duration_s = float(-1) - _create_collector_request.interval_s = float(-1) + _create_collector_request = telemetry_frontend_pb2.Collector() + _create_collector_request.collector_id.collector_id.uuid = str(coll_id_str) + _create_collector_request.kpi_id.kpi_id.uuid = 'KPIid' + str(coll_id_str) + _create_collector_request.duration_s = float(-1) + _create_collector_request.interval_s = float(-1) return _create_collector_request -- GitLab From be791afc3d372eaadb9f7e7471864e05629e2c74 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 16:15:51 +0000 Subject: [PATCH 078/106] files name updation --- ...ts_locally_kpi_manager.sh => run_tests_locally-kpi_manager.sh} | 0 ...emetry-frontend.sh => run_tests_locally-telemetry-frontend.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{run_tests_locally_kpi_manager.sh => run_tests_locally-kpi_manager.sh} (100%) rename scripts/{run_tests_locally_telemetry-frontend.sh => run_tests_locally-telemetry-frontend.sh} (100%) diff --git a/scripts/run_tests_locally_kpi_manager.sh b/scripts/run_tests_locally-kpi_manager.sh similarity index 100% rename from scripts/run_tests_locally_kpi_manager.sh rename to scripts/run_tests_locally-kpi_manager.sh diff --git a/scripts/run_tests_locally_telemetry-frontend.sh b/scripts/run_tests_locally-telemetry-frontend.sh similarity index 100% rename from scripts/run_tests_locally_telemetry-frontend.sh rename to scripts/run_tests_locally-telemetry-frontend.sh -- GitLab From 4343146c456ecd906c0ab51154e5361fb4852a57 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 16:16:43 +0000 Subject: [PATCH 079/106] "StopCollector" and "SelectCollector" methods are added --- .../TelemetryFrontendServiceServicerImpl.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py index 7814107dd..cb66b6b6a 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -14,8 +14,9 @@ import grpc import logging +from common.proto.context_pb2 import Empty from monitoring.service.NameMapping import NameMapping -from common.proto.telemetry_frontend_pb2 import CollectorId, Collector +from common.proto.telemetry_frontend_pb2 import CollectorId, Collector, CollectorFilter, CollectorList from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceServicer @@ -37,4 +38,16 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): # collector_interval = request.interval_s response.collector_id.uuid = _collector_id.collector_id.uuid + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StopCollector(self, request : CollectorId, grpc_context: grpc.ServicerContext # type: ignore + ) -> Empty: # type: ignore + + return Empty() + + def SelectCollectors(self, request : CollectorFilter, contextgrpc_context: grpc.ServicerContext # type: ignore + ) -> CollectorList: # type: ignore + response = CollectorList() + return response \ No newline at end of file -- GitLab From e2c690c1edbe07abae9531f39901c2f0cf8bbbe3 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 16:18:30 +0000 Subject: [PATCH 080/106] "create_collector_filter" method is added in "Messages.py" --- src/telemetry_frontend/tests/Messages.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry_frontend/tests/Messages.py index d63422b72..877781b79 100644 --- a/src/telemetry_frontend/tests/Messages.py +++ b/src/telemetry_frontend/tests/Messages.py @@ -28,3 +28,7 @@ def create_collector_request(coll_id_str): _create_collector_request.duration_s = float(-1) _create_collector_request.interval_s = float(-1) return _create_collector_request + +def create_collector_filter(filter): + _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() + \ No newline at end of file -- GitLab From 231ab1e65b6f4b69c8c65e01b5e3be050273ec84 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 26 Apr 2024 16:19:31 +0000 Subject: [PATCH 081/106] "test_stop_collector" and "test_select_collectors" are added --- src/telemetry_frontend/tests/test_unitary.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/telemetry_frontend/tests/test_unitary.py b/src/telemetry_frontend/tests/test_unitary.py index 4f37514dc..4b2594839 100644 --- a/src/telemetry_frontend/tests/test_unitary.py +++ b/src/telemetry_frontend/tests/test_unitary.py @@ -16,8 +16,9 @@ import os import pytest import logging from typing import Union +from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum -from common.proto.telemetry_frontend_pb2 import CollectorId +from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server from context.client.ContextClient import ContextClient from common.tools.service.GenericGrpcService import GenericGrpcService @@ -167,3 +168,14 @@ def test_start_collector(telemetryFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, CollectorId) +def test_stop_collector(telemetryFrontend_client): + LOGGER.warning('test_stop_collector requesting') + response = telemetryFrontend_client.StopCollector('1') + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +def test_select_collectors(telemetryFrontend_client): + LOGGER.warning('test_select_collector requesting') + response = telemetryFrontend_client.SelectCollectors() + LOGGER.debug(str(response)) + assert isinstance(response, CollectorList) \ No newline at end of file -- GitLab From 269da2e96f88e271fc255375923f437ba7a6342c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 29 Apr 2024 12:36:22 +0000 Subject: [PATCH 082/106] dummy logic is added in "SopCollector" method --- .../service/TelemetryFrontendServiceServicerImpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py index cb66b6b6a..498d07a91 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -43,7 +43,7 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StopCollector(self, request : CollectorId, grpc_context: grpc.ServicerContext # type: ignore ) -> Empty: # type: ignore - + request.collector_id.uuid = "" return Empty() def SelectCollectors(self, request : CollectorFilter, contextgrpc_context: grpc.ServicerContext # type: ignore -- GitLab From 1c177bb7a6d589cdd0d1f38d001299cd8c646628 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 29 Apr 2024 12:37:22 +0000 Subject: [PATCH 083/106] New test case are added --- src/telemetry_frontend/tests/Messages.py | 42 ++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry_frontend/tests/Messages.py index 877781b79..86c869834 100644 --- a/src/telemetry_frontend/tests/Messages.py +++ b/src/telemetry_frontend/tests/Messages.py @@ -13,12 +13,11 @@ # limitations under the License. from common.proto import telemetry_frontend_pb2 -# from common.proto.kpi_manager_pb2 import kpi_id -# from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.proto.kpi_sample_types_pb2 import KpiSampleType -def collector_id(): +def create_collector_id(coll_id_str): _collector_id = telemetry_frontend_pb2.CollectorId() - _collector_id.collector_id.uuid = str(1) + _collector_id.collector_id.uuid = str(coll_id_str) return _collector_id def create_collector_request(coll_id_str): @@ -29,6 +28,37 @@ def create_collector_request(coll_id_str): _create_collector_request.interval_s = float(-1) return _create_collector_request -def create_collector_filter(filter): +def create_collector_request_a(): + _create_collector_request_a = telemetry_frontend_pb2.Collector() + _create_collector_request_a.kpi_id.kpi_id.uuid = "-1" + return _create_collector_request_a + +def create_collector_request_b(str_kpi_id, coll_duration_s, coll_interval_s): + _create_collector_request_b = telemetry_frontend_pb2.Collector() + _create_collector_request_b.kpi_id.kpi_id.uuid = str_kpi_id + _create_collector_request_b.duration_s = coll_duration_s + _create_collector_request_b.interval_s = coll_interval_s + return _create_collector_request_b + +def create_collector_filter(): _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() - \ No newline at end of file + new_collector_id = _create_collector_filter.collector_id.add() + new_collector_id.collector_id.uuid = "COLL1" + new_kpi_id = _create_collector_filter.kpi_id.add() + new_kpi_id.kpi_id.uuid = "KPI1" + new_device_id = _create_collector_filter.device_id.add() + new_device_id.device_uuid.uuid = 'DEV1' + new_service_id = _create_collector_filter.service_id.add() + new_service_id.service_uuid.uuid = 'SERV1' + new_slice_id = _create_collector_filter.slice_id.add() + new_slice_id.slice_uuid.uuid = 'SLC1' + new_endpoint_id = _create_collector_filter.endpoint_id.add() + new_endpoint_id.endpoint_uuid.uuid = 'END1' + new_connection_id = _create_collector_filter.connection_id.add() + new_connection_id.connection_uuid.uuid = 'CON1' + _create_collector_filter.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) + return _create_collector_filter + +def create_collector_list(): + _create_collector_list = telemetry_frontend_pb2.CollectorList() + return _create_collector_list \ No newline at end of file -- GitLab From f6c040ab31901e0049c80aa0d91625dafd12b78c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 29 Apr 2024 12:37:57 +0000 Subject: [PATCH 084/106] New test cases are added --- src/telemetry_frontend/tests/test_unitary.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/telemetry_frontend/tests/test_unitary.py b/src/telemetry_frontend/tests/test_unitary.py index 4b2594839..68467590f 100644 --- a/src/telemetry_frontend/tests/test_unitary.py +++ b/src/telemetry_frontend/tests/test_unitary.py @@ -16,6 +16,7 @@ import os import pytest import logging from typing import Union + from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList @@ -28,7 +29,8 @@ from common.Settings import ( from telemetry_frontend.client.TelemetryFrontendClient import TelemetryFrontendClient from telemetry_frontend.service.TelemetryFrontendService import TelemetryFrontendService -from telemetry_frontend.tests.Messages import create_collector_request +from telemetry_frontend.tests.Messages import ( create_collector_id, create_collector_request, + create_collector_filter, create_collector_request_a, create_collector_request_b) from device.client.DeviceClient import DeviceClient from device.service.DeviceService import DeviceService @@ -168,14 +170,26 @@ def test_start_collector(telemetryFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, CollectorId) +def test_start_collector_a(telemetryFrontend_client): + LOGGER.warning('test_start_collector requesting') + response = telemetryFrontend_client.StartCollector(create_collector_request_a()) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorId) + +def test_start_collector_b(telemetryFrontend_client): + LOGGER.warning('test_start_collector requesting') + response = telemetryFrontend_client.StartCollector(create_collector_request_b('1',10,2)) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorId) + def test_stop_collector(telemetryFrontend_client): LOGGER.warning('test_stop_collector requesting') - response = telemetryFrontend_client.StopCollector('1') + response = telemetryFrontend_client.StopCollector(create_collector_id("1")) LOGGER.debug(str(response)) assert isinstance(response, Empty) def test_select_collectors(telemetryFrontend_client): LOGGER.warning('test_select_collector requesting') - response = telemetryFrontend_client.SelectCollectors() + response = telemetryFrontend_client.SelectCollectors(create_collector_filter()) LOGGER.debug(str(response)) assert isinstance(response, CollectorList) \ No newline at end of file -- GitLab From cd5f2e6565cab03b027614087c409d6431e5fcd2 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:38:28 +0000 Subject: [PATCH 085/106] NodeExporter deplyment script --- manifests/mock_nodeexporter.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 manifests/mock_nodeexporter.yaml diff --git a/manifests/mock_nodeexporter.yaml b/manifests/mock_nodeexporter.yaml new file mode 100644 index 000000000..bf595d63a --- /dev/null +++ b/manifests/mock_nodeexporter.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node-exporter + labels: + app: node-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + containers: + - name: node-exporter + image: prom/node-exporter:latest + ports: + - containerPort: 9100 -- GitLab From 04f1681b12c9d403424434fe6d986c89a65b898c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:38:58 +0000 Subject: [PATCH 086/106] NodeExporter service Script --- manifests/mock_nodeexporterservice.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 manifests/mock_nodeexporterservice.yaml diff --git a/manifests/mock_nodeexporterservice.yaml b/manifests/mock_nodeexporterservice.yaml new file mode 100644 index 000000000..b7bb4f879 --- /dev/null +++ b/manifests/mock_nodeexporterservice.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: node-exporter +spec: + selector: + app: node-exporter + ports: + - protocol: TCP + port: 9100 + targetPort: 9100 + type: NodePort -- GitLab From e08d7ce3e9f51ed989cb7e5a87ce27231a460d40 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:41:24 +0000 Subject: [PATCH 087/106] List of Telemetry virtual enviornment installed packages --- src/telemetry_frontend/telemetry_virenv.txt | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/telemetry_frontend/telemetry_virenv.txt diff --git a/src/telemetry_frontend/telemetry_virenv.txt b/src/telemetry_frontend/telemetry_virenv.txt new file mode 100644 index 000000000..0ce9b803a --- /dev/null +++ b/src/telemetry_frontend/telemetry_virenv.txt @@ -0,0 +1,44 @@ +anytree==2.8.0 +APScheduler==3.10.1 +attrs==23.2.0 +certifi==2024.2.2 +charset-normalizer==2.0.12 +colorama==0.4.6 +confluent-kafka==2.3.0 +coverage==6.3 +future-fstrings==1.2.0 +grpcio==1.47.5 +grpcio-health-checking==1.47.5 +grpcio-tools==1.47.5 +grpclib==0.4.4 +h2==4.1.0 +hpack==4.0.0 +hyperframe==6.0.1 +idna==3.7 +influx-line-protocol==0.1.4 +iniconfig==2.0.0 +kafka-python==2.0.2 +multidict==6.0.5 +networkx==3.3 +packaging==24.0 +pluggy==1.5.0 +prettytable==3.5.0 +prometheus-client==0.13.0 +protobuf==3.20.3 +psycopg2-binary==2.9.3 +py==1.11.0 +py-cpuinfo==9.0.0 +pytest==6.2.5 +pytest-benchmark==3.4.1 +pytest-depends==1.0.1 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +pytz==2024.1 +questdb==1.0.1 +requests==2.27.1 +six==1.16.0 +toml==0.10.2 +tzlocal==5.2 +urllib3==1.26.18 +wcwidth==0.2.13 +xmltodict==0.12.0 -- GitLab From 54839a5ca7635a187bdeb4c7e73c7e249eee6e3c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:43:19 +0000 Subject: [PATCH 088/106] KafkaProducerController: for Kafka configuration and execution of producer --- .../backend/KafkaProducerController.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 src/telemetry_frontend/backend/KafkaProducerController.py diff --git a/src/telemetry_frontend/backend/KafkaProducerController.py b/src/telemetry_frontend/backend/KafkaProducerController.py new file mode 100755 index 000000000..8c88d5e8e --- /dev/null +++ b/src/telemetry_frontend/backend/KafkaProducerController.py @@ -0,0 +1,57 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from NodeExporterProducer import KafkaNodeExporterProducer + +class KafkaProducerController: + """ + Class to control Kafka producer functionality. + """ + def __init__(self): + kafka_configs = self.generate_kafka_configurations() + self.bootstrap_servers = kafka_configs['bootstrap_servers'] + self.node_exporter_endpoint = kafka_configs['node_exporter_endpoint'] + self.kafka_topic = kafka_configs['kafka_topic'] + self.run_duration = kafka_configs['run_duration'] + self.fetch_interval = kafka_configs['fetch_interval'] + + def generate_kafka_configurations(self): + """ + Method to generate Kafka configurations + """ + create_kafka_configuration = { + 'bootstrap_servers' : '127.0.0.1:9092', # Kafka broker address - Replace with your Kafka broker address + 'node_exporter_endpoint' : 'http://10.152.183.231:9100/metrics', # Node Exporter metrics endpoint - Replace with your Node Exporter endpoint + 'kafka_topic' : 'metric-data', # Kafka topic to produce to + 'run_duration' : 20, # Total duration to execute the producer + 'fetch_interval' : 3 # Time between two fetch requests + } + return create_kafka_configuration + + def run_producer(self): + """ + Method to create KafkaNodeExporterProducer object and start producer thread. + """ + # Create NodeExporterProducer object and run start_producer_thread + producer = KafkaNodeExporterProducer(self.bootstrap_servers, self.node_exporter_endpoint, + self.kafka_topic, self.run_duration, self.fetch_interval + ) + # producer.start_producer_thread() # if threading is required + producer.produce_metrics() # if threading is not required + +if __name__ == "__main__": + + # Create Kafka producer controller object and run producer + kafka_controller = KafkaProducerController() + kafka_controller.run_producer() -- GitLab From 179a2193cf887ba4aabd677bd745d148d800c136 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:44:42 +0000 Subject: [PATCH 089/106] NodeExporterProducer: Implementation of Producer: Read from NodeExporter endpoint and writes on Kafka topic --- .../backend/NodeExporterProducer.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 src/telemetry_frontend/backend/NodeExporterProducer.py diff --git a/src/telemetry_frontend/backend/NodeExporterProducer.py b/src/telemetry_frontend/backend/NodeExporterProducer.py new file mode 100755 index 000000000..5f7b1471b --- /dev/null +++ b/src/telemetry_frontend/backend/NodeExporterProducer.py @@ -0,0 +1,128 @@ + +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from confluent_kafka import Producer, KafkaException +from confluent_kafka.admin import AdminClient, NewTopic +import requests +import time +import threading + +class KafkaNodeExporterProducer: + """ + Class to fetch metrics from Node Exporter and produce them to Kafka. + """ + + def __init__(self, bootstrap_servers, node_exporter_endpoint, kafka_topic, run_duration, fetch_interval): + """ + Constructor to initialize Kafka producer parameters. + Args: + bootstrap_servers (str): Kafka broker address. + node_exporter_endpoint (str): Node Exporter metrics endpoint. + kafka_topic (str): Kafka topic to produce metrics to. + run_interval (int): Time interval in seconds to run the producer. + """ + self.bootstrap_servers = bootstrap_servers + self.node_exporter_endpoint = node_exporter_endpoint + self.kafka_topic = kafka_topic + self.run_duration = run_duration + self.fetch_interval = fetch_interval + + def fetch_metrics(self): + """ + Method to fetch metrics from Node Exporter. + Returns: + str: Metrics fetched from Node Exporter. + """ + try: + response = requests.get(self.node_exporter_endpoint) + if response.status_code == 200: + print(f"Metrics fetched sucessfully...") + return response.text + else: + print(f"Failed to fetch metrics. Status code: {response.status_code}") + return None + except Exception as e: + print(f"Failed to fetch metrics: {str(e)}") + return None + + def delivery_callback(self, err, msg): + """ + Callback function to handle message delivery status. + Args: + err (KafkaError): Kafka error object. + msg (Message): Kafka message object. + """ + if err: + print(f'Message delivery failed: {err}') + else: + print(f'Message delivered to topic {msg.topic()}') + + def create_topic_if_not_exists(self, admin_client): + """ + Method to create Kafka topic if it does not exist. + Args: + admin_client (AdminClient): Kafka admin client. + """ + try: + topic_metadata = admin_client.list_topics(timeout=5) + if self.kafka_topic not in topic_metadata.topics: + # If the topic does not exist, create a new topic + print(f"Topic '{self.kafka_topic}' does not exist. Creating...") + new_topic = NewTopic(self.kafka_topic, num_partitions=1, replication_factor=1) + admin_client.create_topics([new_topic]) + except KafkaException as e: + print(f"Failed to create topic: {e}") + + def produce_metrics(self): + """ + Method to continuously produce metrics to Kafka topic. + """ + conf = { + 'bootstrap.servers': self.bootstrap_servers, + } + + admin_client = AdminClient(conf) + self.create_topic_if_not_exists(admin_client) + + kafka_producer = Producer(conf) + + try: + start_time = time.time() + while True: + metrics = self.fetch_metrics() + + if metrics: + kafka_producer.produce(self.kafka_topic, metrics.encode('utf-8'), callback=self.delivery_callback) + kafka_producer.flush() + print("Metrics produced to Kafka topic") + + # Check if the specified run duration has elapsed + if time.time() - start_time >= self.run_duration: + break + + # waiting time until next fetch + time.sleep(self.fetch_interval) + except KeyboardInterrupt: + print("Keyboard interrupt detected. Exiting...") + finally: + kafka_producer.flush() + # kafka_producer.close() # this command generates ERROR + + def start_producer_thread(self): + """ + Method to start the producer thread. + """ + producer_thread = threading.Thread(target=self.produce_metrics) + producer_thread.start() -- GitLab From 97e5fafc9201f56737444996639e9f54f1483e27 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 13:55:07 +0000 Subject: [PATCH 090/106] format improvements --- src/telemetry_frontend/backend/NodeExporterProducer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/telemetry_frontend/backend/NodeExporterProducer.py b/src/telemetry_frontend/backend/NodeExporterProducer.py index 5f7b1471b..b15943727 100755 --- a/src/telemetry_frontend/backend/NodeExporterProducer.py +++ b/src/telemetry_frontend/backend/NodeExporterProducer.py @@ -33,11 +33,11 @@ class KafkaNodeExporterProducer: kafka_topic (str): Kafka topic to produce metrics to. run_interval (int): Time interval in seconds to run the producer. """ - self.bootstrap_servers = bootstrap_servers + self.bootstrap_servers = bootstrap_servers self.node_exporter_endpoint = node_exporter_endpoint - self.kafka_topic = kafka_topic - self.run_duration = run_duration - self.fetch_interval = fetch_interval + self.kafka_topic = kafka_topic + self.run_duration = run_duration + self.fetch_interval = fetch_interval def fetch_metrics(self): """ -- GitLab From 0e46347ccacdd2c76d0a2030a9e6233d7d885695 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 15:43:42 +0000 Subject: [PATCH 091/106] files rename to "KafkaProducerService" and "KafkaProducerServiceImpl" --- .../KafkaProducerService.py} | 14 +++--- .../KafkaProducerServiceImpl.py} | 43 +++++++++++++++---- 2 files changed, 42 insertions(+), 15 deletions(-) rename src/telemetry_frontend/backend/{KafkaProducerController.py => service/KafkaProducerService.py} (83%) rename src/telemetry_frontend/backend/{NodeExporterProducer.py => service/KafkaProducerServiceImpl.py} (70%) diff --git a/src/telemetry_frontend/backend/KafkaProducerController.py b/src/telemetry_frontend/backend/service/KafkaProducerService.py similarity index 83% rename from src/telemetry_frontend/backend/KafkaProducerController.py rename to src/telemetry_frontend/backend/service/KafkaProducerService.py index 8c88d5e8e..fd2507abf 100755 --- a/src/telemetry_frontend/backend/KafkaProducerController.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerService.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from NodeExporterProducer import KafkaNodeExporterProducer +from KafkaProducerServiceImpl import KafkaProducerServiceImpl -class KafkaProducerController: +class KafkaProducerService: """ Class to control Kafka producer functionality. """ @@ -35,16 +35,16 @@ class KafkaProducerController: 'node_exporter_endpoint' : 'http://10.152.183.231:9100/metrics', # Node Exporter metrics endpoint - Replace with your Node Exporter endpoint 'kafka_topic' : 'metric-data', # Kafka topic to produce to 'run_duration' : 20, # Total duration to execute the producer - 'fetch_interval' : 3 # Time between two fetch requests + 'fetch_interval' : 4 # Time between two fetch requests } return create_kafka_configuration def run_producer(self): """ - Method to create KafkaNodeExporterProducer object and start producer thread. + Method to create KafkaProducerServiceImpl object and start producer thread. """ # Create NodeExporterProducer object and run start_producer_thread - producer = KafkaNodeExporterProducer(self.bootstrap_servers, self.node_exporter_endpoint, + producer = KafkaProducerServiceImpl(self.bootstrap_servers, self.node_exporter_endpoint, self.kafka_topic, self.run_duration, self.fetch_interval ) # producer.start_producer_thread() # if threading is required @@ -52,6 +52,6 @@ class KafkaProducerController: if __name__ == "__main__": - # Create Kafka producer controller object and run producer - kafka_controller = KafkaProducerController() + # Create Kafka producer service object and run producer + kafka_controller = KafkaProducerService() kafka_controller.run_producer() diff --git a/src/telemetry_frontend/backend/NodeExporterProducer.py b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py similarity index 70% rename from src/telemetry_frontend/backend/NodeExporterProducer.py rename to src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py index b15943727..6017f26cc 100755 --- a/src/telemetry_frontend/backend/NodeExporterProducer.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py @@ -19,9 +19,9 @@ import requests import time import threading -class KafkaNodeExporterProducer: +class KafkaProducerServiceImpl: """ - Class to fetch metrics from Node Exporter and produce them to Kafka. + Class to fetch metrics from Exporter and produce them to Kafka. """ def __init__(self, bootstrap_servers, node_exporter_endpoint, kafka_topic, run_duration, fetch_interval): @@ -39,17 +39,25 @@ class KafkaNodeExporterProducer: self.run_duration = run_duration self.fetch_interval = fetch_interval - def fetch_metrics(self): + def fetch_node_exporter_metrics(self): """ Method to fetch metrics from Node Exporter. Returns: str: Metrics fetched from Node Exporter. """ + KPI = "node_network_receive_packets_total" try: response = requests.get(self.node_exporter_endpoint) if response.status_code == 200: - print(f"Metrics fetched sucessfully...") - return response.text + # print(f"Metrics fetched sucessfully...") + metrics = response.text + # Check if the desired metric is available in the response + if KPI in metrics: + KPI_VALUE = self.extract_metric_value(metrics, KPI) + # Extract the metric value + if KPI_VALUE is not None: + print(f"KPI value: {KPI_VALUE}") + return KPI_VALUE else: print(f"Failed to fetch metrics. Status code: {response.status_code}") return None @@ -57,6 +65,25 @@ class KafkaNodeExporterProducer: print(f"Failed to fetch metrics: {str(e)}") return None + def extract_metric_value(self, metrics, metric_name): + """ + Method to extract the value of a metric from the metrics string. + Args: + metrics (str): Metrics string fetched from Node Exporter. + metric_name (str): Name of the metric to extract. + Returns: + float: Value of the extracted metric, or None if not found. + """ + try: + # Find the metric line containing the desired metric name + metric_line = next(line for line in metrics.split('\n') if line.startswith(metric_name)) + # Split the line to extract the metric value + metric_value = float(metric_line.split()[1]) + return metric_value + except StopIteration: + print(f"Metric '{metric_name}' not found in the metrics.") + return None + def delivery_callback(self, err, msg): """ Callback function to handle message delivery status. @@ -101,12 +128,12 @@ class KafkaNodeExporterProducer: try: start_time = time.time() while True: - metrics = self.fetch_metrics() + metrics = self.fetch_node_exporter_metrics() # select the function name based on the provided requirements if metrics: - kafka_producer.produce(self.kafka_topic, metrics.encode('utf-8'), callback=self.delivery_callback) + kafka_producer.produce(self.kafka_topic, str(metrics), callback=self.delivery_callback) kafka_producer.flush() - print("Metrics produced to Kafka topic") + # print("Metrics produced to Kafka topic") # Check if the specified run duration has elapsed if time.time() - start_time >= self.run_duration: -- GitLab From a5f83a179afd7b4176f4d72befdd475af839332a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Apr 2024 15:44:08 +0000 Subject: [PATCH 092/106] Kafka tests file added --- .../backend/tests/KafkaProducerTests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/telemetry_frontend/backend/tests/KafkaProducerTests.py diff --git a/src/telemetry_frontend/backend/tests/KafkaProducerTests.py b/src/telemetry_frontend/backend/tests/KafkaProducerTests.py new file mode 100644 index 000000000..b353f9fe2 --- /dev/null +++ b/src/telemetry_frontend/backend/tests/KafkaProducerTests.py @@ -0,0 +1,19 @@ + +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from telemetry_frontend.backend.service.KafkaProducerService import KafkaProducerService + +kafka_controller = KafkaProducerService() +kafka_controller.run_producer() \ No newline at end of file -- GitLab From 23742187793fe472292300d972b7a07ba454d827 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:01:42 +0000 Subject: [PATCH 093/106] Script to run "Telemetry Backend" service --- .../run_tests_locally-telemetry-backend.sh | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 scripts/run_tests_locally-telemetry-backend.sh diff --git a/scripts/run_tests_locally-telemetry-backend.sh b/scripts/run_tests_locally-telemetry-backend.sh new file mode 100755 index 000000000..cbebd6807 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-backend.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --verbose \ + telemetry_frontend/backend/tests/test_kafka_backend.py \ No newline at end of file -- GitLab From 6d5bc62a7cb4c87315fa4c23068e45babf8e409b Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:02:12 +0000 Subject: [PATCH 094/106] __init__.py file added --- src/telemetry_frontend/__init__.py | 16 +++++++++++++++- .../{tests/KafkaProducerTests.py => __init__.py} | 6 ------ .../backend/service/__init__.py | 13 +++++++++++++ src/telemetry_frontend/backend/tests/__init__.py | 13 +++++++++++++ 4 files changed, 41 insertions(+), 7 deletions(-) rename src/telemetry_frontend/backend/{tests/KafkaProducerTests.py => __init__.py} (78%) create mode 100644 src/telemetry_frontend/backend/service/__init__.py create mode 100644 src/telemetry_frontend/backend/tests/__init__.py diff --git a/src/telemetry_frontend/__init__.py b/src/telemetry_frontend/__init__.py index eb1ae458f..6a8f39746 100644 --- a/src/telemetry_frontend/__init__.py +++ b/src/telemetry_frontend/__init__.py @@ -1 +1,15 @@ -... + +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry_frontend/backend/tests/KafkaProducerTests.py b/src/telemetry_frontend/backend/__init__.py similarity index 78% rename from src/telemetry_frontend/backend/tests/KafkaProducerTests.py rename to src/telemetry_frontend/backend/__init__.py index b353f9fe2..38d04994f 100644 --- a/src/telemetry_frontend/backend/tests/KafkaProducerTests.py +++ b/src/telemetry_frontend/backend/__init__.py @@ -1,4 +1,3 @@ - # Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,8 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from telemetry_frontend.backend.service.KafkaProducerService import KafkaProducerService - -kafka_controller = KafkaProducerService() -kafka_controller.run_producer() \ No newline at end of file diff --git a/src/telemetry_frontend/backend/service/__init__.py b/src/telemetry_frontend/backend/service/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/telemetry_frontend/backend/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/telemetry_frontend/backend/tests/__init__.py b/src/telemetry_frontend/backend/tests/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/telemetry_frontend/backend/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -- GitLab From beb53c88457021589a6a6a12288120ab9aaba74a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:03:24 +0000 Subject: [PATCH 095/106] method name change from "generate_kafka_configurations" to "generate_kafka_configs" --- .../backend/service/KafkaProducerService.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/telemetry_frontend/backend/service/KafkaProducerService.py b/src/telemetry_frontend/backend/service/KafkaProducerService.py index fd2507abf..4e2d79347 100755 --- a/src/telemetry_frontend/backend/service/KafkaProducerService.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerService.py @@ -12,36 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -from KafkaProducerServiceImpl import KafkaProducerServiceImpl +from .KafkaProducerServiceImpl import KafkaProducerServiceImpl class KafkaProducerService: """ Class to control Kafka producer functionality. """ def __init__(self): - kafka_configs = self.generate_kafka_configurations() + + kafka_configs = self.generate_kafka_configs() + self.bootstrap_servers = kafka_configs['bootstrap_servers'] self.node_exporter_endpoint = kafka_configs['node_exporter_endpoint'] self.kafka_topic = kafka_configs['kafka_topic'] self.run_duration = kafka_configs['run_duration'] self.fetch_interval = kafka_configs['fetch_interval'] - def generate_kafka_configurations(self): + def generate_kafka_configs(self): # define the function to get every attribute """ Method to generate Kafka configurations """ - create_kafka_configuration = { + create_kafka_configs = { 'bootstrap_servers' : '127.0.0.1:9092', # Kafka broker address - Replace with your Kafka broker address 'node_exporter_endpoint' : 'http://10.152.183.231:9100/metrics', # Node Exporter metrics endpoint - Replace with your Node Exporter endpoint 'kafka_topic' : 'metric-data', # Kafka topic to produce to 'run_duration' : 20, # Total duration to execute the producer 'fetch_interval' : 4 # Time between two fetch requests } - return create_kafka_configuration + return create_kafka_configs def run_producer(self): """ - Method to create KafkaProducerServiceImpl object and start producer thread. + Method to create KafkaProducerServiceImpl object and start producer. """ # Create NodeExporterProducer object and run start_producer_thread producer = KafkaProducerServiceImpl(self.bootstrap_servers, self.node_exporter_endpoint, -- GitLab From 7a1da84510d9aa5480612edaf04fe311ffda63bd Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:05:00 +0000 Subject: [PATCH 096/106] Two methods add "export_collector_value()" and "write_to_kafka()" --- .../service/KafkaProducerServiceImpl.py | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py index 6017f26cc..b6b55f913 100755 --- a/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py @@ -13,11 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -from confluent_kafka import Producer, KafkaException -from confluent_kafka.admin import AdminClient, NewTopic -import requests import time +import grpc +import logging +import requests import threading +from common.proto.context_pb2 import Empty +from confluent_kafka import Producer, KafkaException +from confluent_kafka.admin import AdminClient, NewTopic +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('Telemetry', 'TelemetryBackend') class KafkaProducerServiceImpl: """ @@ -33,11 +41,25 @@ class KafkaProducerServiceImpl: kafka_topic (str): Kafka topic to produce metrics to. run_interval (int): Time interval in seconds to run the producer. """ + LOGGER.info('Init TelemetryBackendService') + self.bootstrap_servers = bootstrap_servers self.node_exporter_endpoint = node_exporter_endpoint self.kafka_topic = kafka_topic self.run_duration = run_duration self.fetch_interval = fetch_interval + + # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def export_collector_value(request: CollectorId) -> str: # type: ignore + response = str() + response = '-1' + return response + + # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def write_to_kafka(Collector, kpi_value) -> Empty: # type: ignore + return Empty() + +# ----------- BELOW: Actual Implementation of Kafka Producer with Node Exporter ----------- def fetch_node_exporter_metrics(self): """ @@ -114,7 +136,7 @@ class KafkaProducerServiceImpl: def produce_metrics(self): """ - Method to continuously produce metrics to Kafka topic. + Method to produce metrics to Kafka topic as per Kafka configs. """ conf = { 'bootstrap.servers': self.bootstrap_servers, @@ -146,10 +168,12 @@ class KafkaProducerServiceImpl: finally: kafka_producer.flush() # kafka_producer.close() # this command generates ERROR - + # --- def start_producer_thread(self): """ Method to start the producer thread. """ producer_thread = threading.Thread(target=self.produce_metrics) producer_thread.start() + +# ----------- ABOVE: Actual Implementation of Kafka Producer with Node Exporter ----------- \ No newline at end of file -- GitLab From 062cc59b7e178ab67507fc4d29d09694a0a26bf4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:05:37 +0000 Subject: [PATCH 097/106] "TelemetryBackend" messages file added --- .../backend/tests/messagesBackend.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/telemetry_frontend/backend/tests/messagesBackend.py diff --git a/src/telemetry_frontend/backend/tests/messagesBackend.py b/src/telemetry_frontend/backend/tests/messagesBackend.py new file mode 100644 index 000000000..bfcebf0cc --- /dev/null +++ b/src/telemetry_frontend/backend/tests/messagesBackend.py @@ -0,0 +1,46 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_kafka_config(): + """ + No input parameter is requested + Returns the dict object with Kafka configs + """ + _kafka_configs = dict() + _kafka_configs['bootstrap_servers'] = '127.0.0.1:9092' + _kafka_configs['exporter_endpoint'] = 'http://10.152.183.231:9100/metrics' + _kafka_configs['kafka_topic'] = 'metric-data' + _kafka_configs['run_duration'] = 20 + _kafka_configs['fetch_interval'] = 4 + + return _kafka_configs + +def create_kafka_config_a(bootstrap_server, exporter_endpoint, kafka_topic, run_duration, fetch_interval): + """ + Provide ... + Bootstrap_server IP address as String. + Exporter endpoint with port address as String. + Kafka topic name as String. + Total duration of the test as Float. + Fetch_interval as Float. + """ + _kafka_configs = dict() + _kafka_configs['bootstrap_servers'] = bootstrap_server + _kafka_configs['exporter_endpoint'] = exporter_endpoint + _kafka_configs['kafka_topic'] = kafka_topic + _kafka_configs['run_duration'] = run_duration + _kafka_configs['fetch_interval'] = fetch_interval + + return _kafka_configs \ No newline at end of file -- GitLab From 4b39f8d0b7a2d553b670d3a9f28d3e3a420f61bc Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 7 May 2024 14:06:53 +0000 Subject: [PATCH 098/106] "TelemetryBackend" tests "test_get_kafka_configs()", "test_get_kafka_configs_a()" and "test_export_collector_value()" are added. --- .../backend/tests/test_kafka_backend.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/telemetry_frontend/backend/tests/test_kafka_backend.py diff --git a/src/telemetry_frontend/backend/tests/test_kafka_backend.py b/src/telemetry_frontend/backend/tests/test_kafka_backend.py new file mode 100644 index 000000000..5caabb9e0 --- /dev/null +++ b/src/telemetry_frontend/backend/tests/test_kafka_backend.py @@ -0,0 +1,51 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# import sys +# print (sys.path) +import logging +from .messagesBackend import create_kafka_config, create_kafka_config_a +from src.telemetry_frontend.tests.Messages import create_collector_request +from src.telemetry_frontend.backend.service.KafkaProducerService import KafkaProducerService +from src.telemetry_frontend.backend.service.KafkaProducerServiceImpl import KafkaProducerServiceImpl + +LOGGER = logging.getLogger(__name__) + + +########################### +# Tests Implementation of Telemetry Backend +########################### +def test_get_kafka_configs(): + LOGGER.warning('test_get_kafka_configs requesting') + response = KafkaProducerService.generate_kafka_configs( + create_kafka_config() + ) + LOGGER.debug(str(response)) + assert isinstance(response, dict) + +def test_get_kafka_configs_a(): + LOGGER.warning('test_get_kafka_configs_a requesting') + response = KafkaProducerService.generate_kafka_configs( + create_kafka_config_a('ip:port', 'ip:port', 'test_topic', 10, 3) + ) + LOGGER.debug(str(response)) + assert isinstance(response, dict) + +def test_export_collector_value(): + LOGGER.warning('test_export_collector_value requesting') + response = KafkaProducerServiceImpl.export_collector_value( + create_collector_request('1') + ) + LOGGER.debug(str(response)) + assert isinstance(response, str) \ No newline at end of file -- GitLab From 1c03799fdf4730e2a6a389a3d9c3d82440b77ac9 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:24:56 +0000 Subject: [PATCH 099/106] Move "Kafka_configs" initiation to "generate_kafka_configs" --- .../backend/service/KafkaProducerService.py | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/telemetry_frontend/backend/service/KafkaProducerService.py b/src/telemetry_frontend/backend/service/KafkaProducerService.py index 4e2d79347..0a76b2d99 100755 --- a/src/telemetry_frontend/backend/service/KafkaProducerService.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerService.py @@ -19,25 +19,19 @@ class KafkaProducerService: Class to control Kafka producer functionality. """ def __init__(self): + pass - kafka_configs = self.generate_kafka_configs() - self.bootstrap_servers = kafka_configs['bootstrap_servers'] - self.node_exporter_endpoint = kafka_configs['node_exporter_endpoint'] - self.kafka_topic = kafka_configs['kafka_topic'] - self.run_duration = kafka_configs['run_duration'] - self.fetch_interval = kafka_configs['fetch_interval'] - - def generate_kafka_configs(self): # define the function to get every attribute + def generate_kafka_configs(self): """ Method to generate Kafka configurations """ create_kafka_configs = { - 'bootstrap_servers' : '127.0.0.1:9092', # Kafka broker address - Replace with your Kafka broker address - 'node_exporter_endpoint' : 'http://10.152.183.231:9100/metrics', # Node Exporter metrics endpoint - Replace with your Node Exporter endpoint - 'kafka_topic' : 'metric-data', # Kafka topic to produce to - 'run_duration' : 20, # Total duration to execute the producer - 'fetch_interval' : 4 # Time between two fetch requests + 'bootstrap_servers' : "test_server", # Kafka broker address - Replace with your Kafka broker address + 'exporter_endpoint' : "test_exporter", # Node Exporter metrics endpoint - Replace with your Node Exporter endpoint + 'kafka_topic' : "test_kafka_topic", # Kafka topic to produce to + 'run_duration' : 10, # Total duration to execute the producer + 'fetch_interval' : 2 # Time between two fetch requests } return create_kafka_configs -- GitLab From eb47527c6a668ce8d48086d8f9432c6ad949a19e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:28:28 +0000 Subject: [PATCH 100/106] update methods "export_collector_value" and "write_to_kafka" parameters and return type of "export_collector_value". --- .../service/KafkaProducerServiceImpl.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py index b6b55f913..da5513170 100755 --- a/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py +++ b/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py @@ -14,10 +14,10 @@ # limitations under the License. import time -import grpc import logging import requests import threading +from typing import Tuple from common.proto.context_pb2 import Empty from confluent_kafka import Producer, KafkaException from confluent_kafka.admin import AdminClient, NewTopic @@ -32,31 +32,34 @@ class KafkaProducerServiceImpl: Class to fetch metrics from Exporter and produce them to Kafka. """ - def __init__(self, bootstrap_servers, node_exporter_endpoint, kafka_topic, run_duration, fetch_interval): + def __init__(self, bootstrap_servers=None, exporter_endpoint=None, + kafka_topic=None, run_duration=None, fetch_interval=None): """ Constructor to initialize Kafka producer parameters. Args: bootstrap_servers (str): Kafka broker address. - node_exporter_endpoint (str): Node Exporter metrics endpoint. + exporter_endpoint (str): Node Exporter metrics endpoint. kafka_topic (str): Kafka topic to produce metrics to. run_interval (int): Time interval in seconds to run the producer. """ LOGGER.info('Init TelemetryBackendService') - self.bootstrap_servers = bootstrap_servers - self.node_exporter_endpoint = node_exporter_endpoint - self.kafka_topic = kafka_topic - self.run_duration = run_duration - self.fetch_interval = fetch_interval + self.bootstrap_servers = bootstrap_servers + self.exporter_endpoint = exporter_endpoint + self.kafka_topic = kafka_topic + self.run_duration = run_duration + self.fetch_interval = fetch_interval # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def export_collector_value(request: CollectorId) -> str: # type: ignore - response = str() - response = '-1' + def export_collector_value(request : Collector) -> Tuple[str, str]: # type: ignore + response = Tuple[str, str] + response = ('test collector Id', 'test collected value') # Metric to be fetched from endpoint based on Collector message return response # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def write_to_kafka(Collector, kpi_value) -> Empty: # type: ignore + def write_to_kafka(request: Tuple[str, str]) -> Empty: # type: ignore + # _collector_id, _collector_id_value = request + # write collector_id and collector_id value on the Kafka topic return Empty() # ----------- BELOW: Actual Implementation of Kafka Producer with Node Exporter ----------- @@ -69,7 +72,7 @@ class KafkaProducerServiceImpl: """ KPI = "node_network_receive_packets_total" try: - response = requests.get(self.node_exporter_endpoint) + response = requests.get(self.exporter_endpoint) if response.status_code == 200: # print(f"Metrics fetched sucessfully...") metrics = response.text -- GitLab From a7e1ddfbc2ee39936c83f2d6a2105ebfd97473bd Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:29:03 +0000 Subject: [PATCH 101/106] Remove "create_kafka_config" message type --- .../backend/tests/messagesBackend.py | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/telemetry_frontend/backend/tests/messagesBackend.py b/src/telemetry_frontend/backend/tests/messagesBackend.py index bfcebf0cc..ef1235383 100644 --- a/src/telemetry_frontend/backend/tests/messagesBackend.py +++ b/src/telemetry_frontend/backend/tests/messagesBackend.py @@ -13,34 +13,20 @@ # limitations under the License. -def create_kafka_config(): - """ - No input parameter is requested - Returns the dict object with Kafka configs - """ - _kafka_configs = dict() - _kafka_configs['bootstrap_servers'] = '127.0.0.1:9092' - _kafka_configs['exporter_endpoint'] = 'http://10.152.183.231:9100/metrics' - _kafka_configs['kafka_topic'] = 'metric-data' - _kafka_configs['run_duration'] = 20 - _kafka_configs['fetch_interval'] = 4 - - return _kafka_configs - -def create_kafka_config_a(bootstrap_server, exporter_endpoint, kafka_topic, run_duration, fetch_interval): +def create_kafka_config_a(bootstrap_server: str, exporter_endpoint: str, kafka_topic: str, + run_duration: int, fetch_interval: int): """ Provide ... Bootstrap_server IP address as String. Exporter endpoint with port address as String. Kafka topic name as String. - Total duration of the test as Float. - Fetch_interval as Float. + Total duration of the test as Int. + Fetch_interval as Int. """ - _kafka_configs = dict() - _kafka_configs['bootstrap_servers'] = bootstrap_server - _kafka_configs['exporter_endpoint'] = exporter_endpoint - _kafka_configs['kafka_topic'] = kafka_topic - _kafka_configs['run_duration'] = run_duration - _kafka_configs['fetch_interval'] = fetch_interval + _bootstrap_servers = bootstrap_server + _exporter_endpoint = exporter_endpoint + _kafka_topic = kafka_topic + _run_duration = run_duration + _fetch_interval = fetch_interval - return _kafka_configs \ No newline at end of file + return _bootstrap_servers, _exporter_endpoint, _kafka_topic, _run_duration, _fetch_interval -- GitLab From f82e44099a206d323c4415605527deb96d040ee2 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:30:06 +0000 Subject: [PATCH 102/106] Test "test_get_kafka_configs_a" is removed and "test_write_to_kafka" is added --- .../backend/tests/test_kafka_backend.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/telemetry_frontend/backend/tests/test_kafka_backend.py b/src/telemetry_frontend/backend/tests/test_kafka_backend.py index 5caabb9e0..e64a65ccd 100644 --- a/src/telemetry_frontend/backend/tests/test_kafka_backend.py +++ b/src/telemetry_frontend/backend/tests/test_kafka_backend.py @@ -15,8 +15,9 @@ # import sys # print (sys.path) import logging -from .messagesBackend import create_kafka_config, create_kafka_config_a -from src.telemetry_frontend.tests.Messages import create_collector_request +from typing import Tuple +from common.proto.context_pb2 import Empty +from src.telemetry_frontend.tests.Messages import create_collector_request, create_collector_id from src.telemetry_frontend.backend.service.KafkaProducerService import KafkaProducerService from src.telemetry_frontend.backend.service.KafkaProducerServiceImpl import KafkaProducerServiceImpl @@ -28,17 +29,8 @@ LOGGER = logging.getLogger(__name__) ########################### def test_get_kafka_configs(): LOGGER.warning('test_get_kafka_configs requesting') - response = KafkaProducerService.generate_kafka_configs( - create_kafka_config() - ) - LOGGER.debug(str(response)) - assert isinstance(response, dict) - -def test_get_kafka_configs_a(): - LOGGER.warning('test_get_kafka_configs_a requesting') - response = KafkaProducerService.generate_kafka_configs( - create_kafka_config_a('ip:port', 'ip:port', 'test_topic', 10, 3) - ) + KafkaProducerServiceObj = KafkaProducerService() + response = KafkaProducerServiceObj.generate_kafka_configs() LOGGER.debug(str(response)) assert isinstance(response, dict) @@ -48,4 +40,11 @@ def test_export_collector_value(): create_collector_request('1') ) LOGGER.debug(str(response)) - assert isinstance(response, str) \ No newline at end of file + assert isinstance(response, Tuple) + +def test_write_to_kafka(): + LOGGER.warning('test_write_to_kafka requesting') + collector_value = KafkaProducerServiceImpl.export_collector_value(create_collector_request('1')) + response = KafkaProducerServiceImpl.write_to_kafka(collector_value) # type: ignore (don't know why warning here) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) -- GitLab From 05194f29c452c8db23ce30443c92f9da05ce1026 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:32:46 +0000 Subject: [PATCH 103/106] Added parameter type in methods "create_collector_id" and "create_collector_request". Replaced "kpi_id.kpi_id.uuid" with "collector_id.collector_id.uuid" in methods "create_collector_request_a" and "create_collector_request_b". --- src/telemetry_frontend/tests/Messages.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry_frontend/tests/Messages.py index 86c869834..d323aa7fd 100644 --- a/src/telemetry_frontend/tests/Messages.py +++ b/src/telemetry_frontend/tests/Messages.py @@ -15,12 +15,12 @@ from common.proto import telemetry_frontend_pb2 from common.proto.kpi_sample_types_pb2 import KpiSampleType -def create_collector_id(coll_id_str): +def create_collector_id(coll_id_str : str): _collector_id = telemetry_frontend_pb2.CollectorId() _collector_id.collector_id.uuid = str(coll_id_str) return _collector_id -def create_collector_request(coll_id_str): +def create_collector_request(coll_id_str : str): _create_collector_request = telemetry_frontend_pb2.Collector() _create_collector_request.collector_id.collector_id.uuid = str(coll_id_str) _create_collector_request.kpi_id.kpi_id.uuid = 'KPIid' + str(coll_id_str) @@ -29,15 +29,16 @@ def create_collector_request(coll_id_str): return _create_collector_request def create_collector_request_a(): - _create_collector_request_a = telemetry_frontend_pb2.Collector() - _create_collector_request_a.kpi_id.kpi_id.uuid = "-1" + _create_collector_request_a = telemetry_frontend_pb2.Collector() + _create_collector_request_a.collector_id.collector_id.uuid = "-1" return _create_collector_request_a def create_collector_request_b(str_kpi_id, coll_duration_s, coll_interval_s): - _create_collector_request_b = telemetry_frontend_pb2.Collector() - _create_collector_request_b.kpi_id.kpi_id.uuid = str_kpi_id - _create_collector_request_b.duration_s = coll_duration_s - _create_collector_request_b.interval_s = coll_interval_s + _create_collector_request_b = telemetry_frontend_pb2.Collector() + _create_collector_request_b.collector_id.collector_id.uuid = '-1' + _create_collector_request_b.kpi_id.kpi_id.uuid = str_kpi_id + _create_collector_request_b.duration_s = coll_duration_s + _create_collector_request_b.interval_s = coll_interval_s return _create_collector_request_b def create_collector_filter(): -- GitLab From 0bcb839ea4cf561431b3a7111de4bde13e942fdb Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:52:53 +0000 Subject: [PATCH 104/106] Two imports were missing --- src/telemetry/frontend/service/__main__.py | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/telemetry/frontend/service/__main__.py diff --git a/src/telemetry/frontend/service/__main__.py b/src/telemetry/frontend/service/__main__.py new file mode 100644 index 000000000..afc381e09 --- /dev/null +++ b/src/telemetry/frontend/service/__main__.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import sys +import logging, threading +from .TelemetryFrontendService import TelemetryFrontendService +from monitoring.service.NameMapping import NameMapping +from monitoring.service.EventTools import EventsDeviceCollector +from common.Settings import ( + get_log_level, wait_for_environment_variables, get_env_var_name, + get_metrics_port ) + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER + + log_level = get_log_level() + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + LOGGER = logging.getLogger(__name__) + +# ------- will be added later -------------- + # wait_for_environment_variables([ + # get_env_var_name + + + # ]) +# ------- will be added later -------------- + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + + grpc_service = TelemetryFrontendService(name_mapping) + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file -- GitLab From f05d40206e1137c3d10de79b5203bd981d8ff87f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 13:55:10 +0000 Subject: [PATCH 105/106] folder name changed from "telemetry_frontend" to "frontend", and "client", "service" and "tests" move inside the "frontend" --- .../__init__.py | 0 .../backend/__init__.py | 0 .../backend/service/KafkaProducerService.py | 0 .../service/KafkaProducerServiceImpl.py | 0 .../backend/service/__init__.py | 0 .../backend/tests/__init__.py | 0 .../backend/tests/messagesBackend.py | 0 .../backend/tests/test_kafka_backend.py | 6 +- src/telemetry/frontend/__init__.py | 15 ++++ .../client/TelemetryFrontendClient.py | 0 .../frontend}/client/__init__.py | 0 .../service/TelemetryFrontendService.py | 2 +- .../TelemetryFrontendServiceServicerImpl.py | 0 .../frontend}/service/__init__.py | 0 .../frontend}/tests/Messages.py | 0 .../frontend}/tests/__init__.py | 0 .../frontend}/tests/test_unitary.py | 6 +- .../requirements.in | 0 .../telemetry_virenv.txt | 0 src/telemetry_frontend/service/__main__.py | 69 ------------------- 20 files changed, 22 insertions(+), 76 deletions(-) rename src/{telemetry_frontend => telemetry}/__init__.py (100%) rename src/{telemetry_frontend => telemetry}/backend/__init__.py (100%) rename src/{telemetry_frontend => telemetry}/backend/service/KafkaProducerService.py (100%) rename src/{telemetry_frontend => telemetry}/backend/service/KafkaProducerServiceImpl.py (100%) rename src/{telemetry_frontend => telemetry}/backend/service/__init__.py (100%) rename src/{telemetry_frontend => telemetry}/backend/tests/__init__.py (100%) rename src/{telemetry_frontend => telemetry}/backend/tests/messagesBackend.py (100%) rename src/{telemetry_frontend => telemetry}/backend/tests/test_kafka_backend.py (87%) create mode 100644 src/telemetry/frontend/__init__.py rename src/{telemetry_frontend => telemetry/frontend}/client/TelemetryFrontendClient.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/client/__init__.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/service/TelemetryFrontendService.py (95%) rename src/{telemetry_frontend => telemetry/frontend}/service/TelemetryFrontendServiceServicerImpl.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/service/__init__.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/tests/Messages.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/tests/__init__.py (100%) rename src/{telemetry_frontend => telemetry/frontend}/tests/test_unitary.py (96%) rename src/{telemetry_frontend => telemetry}/requirements.in (100%) rename src/{telemetry_frontend => telemetry}/telemetry_virenv.txt (100%) delete mode 100644 src/telemetry_frontend/service/__main__.py diff --git a/src/telemetry_frontend/__init__.py b/src/telemetry/__init__.py similarity index 100% rename from src/telemetry_frontend/__init__.py rename to src/telemetry/__init__.py diff --git a/src/telemetry_frontend/backend/__init__.py b/src/telemetry/backend/__init__.py similarity index 100% rename from src/telemetry_frontend/backend/__init__.py rename to src/telemetry/backend/__init__.py diff --git a/src/telemetry_frontend/backend/service/KafkaProducerService.py b/src/telemetry/backend/service/KafkaProducerService.py similarity index 100% rename from src/telemetry_frontend/backend/service/KafkaProducerService.py rename to src/telemetry/backend/service/KafkaProducerService.py diff --git a/src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py b/src/telemetry/backend/service/KafkaProducerServiceImpl.py similarity index 100% rename from src/telemetry_frontend/backend/service/KafkaProducerServiceImpl.py rename to src/telemetry/backend/service/KafkaProducerServiceImpl.py diff --git a/src/telemetry_frontend/backend/service/__init__.py b/src/telemetry/backend/service/__init__.py similarity index 100% rename from src/telemetry_frontend/backend/service/__init__.py rename to src/telemetry/backend/service/__init__.py diff --git a/src/telemetry_frontend/backend/tests/__init__.py b/src/telemetry/backend/tests/__init__.py similarity index 100% rename from src/telemetry_frontend/backend/tests/__init__.py rename to src/telemetry/backend/tests/__init__.py diff --git a/src/telemetry_frontend/backend/tests/messagesBackend.py b/src/telemetry/backend/tests/messagesBackend.py similarity index 100% rename from src/telemetry_frontend/backend/tests/messagesBackend.py rename to src/telemetry/backend/tests/messagesBackend.py diff --git a/src/telemetry_frontend/backend/tests/test_kafka_backend.py b/src/telemetry/backend/tests/test_kafka_backend.py similarity index 87% rename from src/telemetry_frontend/backend/tests/test_kafka_backend.py rename to src/telemetry/backend/tests/test_kafka_backend.py index e64a65ccd..05174da2b 100644 --- a/src/telemetry_frontend/backend/tests/test_kafka_backend.py +++ b/src/telemetry/backend/tests/test_kafka_backend.py @@ -17,9 +17,9 @@ import logging from typing import Tuple from common.proto.context_pb2 import Empty -from src.telemetry_frontend.tests.Messages import create_collector_request, create_collector_id -from src.telemetry_frontend.backend.service.KafkaProducerService import KafkaProducerService -from src.telemetry_frontend.backend.service.KafkaProducerServiceImpl import KafkaProducerServiceImpl +from src.telemetry.frontend.tests.Messages import create_collector_request, create_collector_id +from src.telemetry.backend.service.KafkaProducerService import KafkaProducerService +from src.telemetry.backend.service.KafkaProducerServiceImpl import KafkaProducerServiceImpl LOGGER = logging.getLogger(__name__) diff --git a/src/telemetry/frontend/__init__.py b/src/telemetry/frontend/__init__.py new file mode 100644 index 000000000..6a8f39746 --- /dev/null +++ b/src/telemetry/frontend/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry_frontend/client/TelemetryFrontendClient.py b/src/telemetry/frontend/client/TelemetryFrontendClient.py similarity index 100% rename from src/telemetry_frontend/client/TelemetryFrontendClient.py rename to src/telemetry/frontend/client/TelemetryFrontendClient.py diff --git a/src/telemetry_frontend/client/__init__.py b/src/telemetry/frontend/client/__init__.py similarity index 100% rename from src/telemetry_frontend/client/__init__.py rename to src/telemetry/frontend/client/__init__.py diff --git a/src/telemetry_frontend/service/TelemetryFrontendService.py b/src/telemetry/frontend/service/TelemetryFrontendService.py similarity index 95% rename from src/telemetry_frontend/service/TelemetryFrontendService.py rename to src/telemetry/frontend/service/TelemetryFrontendService.py index a0ae704d3..522d125e6 100644 --- a/src/telemetry_frontend/service/TelemetryFrontendService.py +++ b/src/telemetry/frontend/service/TelemetryFrontendService.py @@ -17,7 +17,7 @@ from common.Settings import get_service_port_grpc from monitoring.service.NameMapping import NameMapping from common.tools.service.GenericGrpcService import GenericGrpcService from common.proto.telemetry_frontend_pb2_grpc import add_TelemetryFrontendServiceServicer_to_server -from telemetry_frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl +from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl class TelemetryFrontendService(GenericGrpcService): diff --git a/src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py similarity index 100% rename from src/telemetry_frontend/service/TelemetryFrontendServiceServicerImpl.py rename to src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py diff --git a/src/telemetry_frontend/service/__init__.py b/src/telemetry/frontend/service/__init__.py similarity index 100% rename from src/telemetry_frontend/service/__init__.py rename to src/telemetry/frontend/service/__init__.py diff --git a/src/telemetry_frontend/tests/Messages.py b/src/telemetry/frontend/tests/Messages.py similarity index 100% rename from src/telemetry_frontend/tests/Messages.py rename to src/telemetry/frontend/tests/Messages.py diff --git a/src/telemetry_frontend/tests/__init__.py b/src/telemetry/frontend/tests/__init__.py similarity index 100% rename from src/telemetry_frontend/tests/__init__.py rename to src/telemetry/frontend/tests/__init__.py diff --git a/src/telemetry_frontend/tests/test_unitary.py b/src/telemetry/frontend/tests/test_unitary.py similarity index 96% rename from src/telemetry_frontend/tests/test_unitary.py rename to src/telemetry/frontend/tests/test_unitary.py index 68467590f..312695659 100644 --- a/src/telemetry_frontend/tests/test_unitary.py +++ b/src/telemetry/frontend/tests/test_unitary.py @@ -27,9 +27,9 @@ from common.tests.MockServicerImpl_Context import MockServicerImpl_Context from common.Settings import ( get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) -from telemetry_frontend.client.TelemetryFrontendClient import TelemetryFrontendClient -from telemetry_frontend.service.TelemetryFrontendService import TelemetryFrontendService -from telemetry_frontend.tests.Messages import ( create_collector_id, create_collector_request, +from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient +from telemetry.frontend.service.TelemetryFrontendService import TelemetryFrontendService +from telemetry.frontend.tests.Messages import ( create_collector_id, create_collector_request, create_collector_filter, create_collector_request_a, create_collector_request_b) from device.client.DeviceClient import DeviceClient diff --git a/src/telemetry_frontend/requirements.in b/src/telemetry/requirements.in similarity index 100% rename from src/telemetry_frontend/requirements.in rename to src/telemetry/requirements.in diff --git a/src/telemetry_frontend/telemetry_virenv.txt b/src/telemetry/telemetry_virenv.txt similarity index 100% rename from src/telemetry_frontend/telemetry_virenv.txt rename to src/telemetry/telemetry_virenv.txt diff --git a/src/telemetry_frontend/service/__main__.py b/src/telemetry_frontend/service/__main__.py deleted file mode 100644 index 9b5fe70de..000000000 --- a/src/telemetry_frontend/service/__main__.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, threading -from .TelemetryFrontendService import TelemetryFrontendService -from monitoring.service.NameMapping import NameMapping -from monitoring.service.EventTools import EventsDeviceCollector -from common.Settings import ( - get_log_level, wait_for_environment_variables, get_env_var_name, - get_metrics_port ) - -terminate = threading.Event() -LOGGER = None - -def signal_handler(signal, frame): # pylint: disable=redefined-outer-name - LOGGER.warning('Terminate signal received') - terminate.set() - -def main(): - global LOGGER - - log_level = get_log_level() - logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") - LOGGER = logging.getLogger(__name__) - -# ------- will be added later -------------- - # wait_for_environment_variables([ - # get_env_var_name - - - # ]) -# ------- will be added later -------------- - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - LOGGER.info('Starting...') - - # Start metrics server - metrics_port = get_metrics_port() - start_http_server(metrics_port) - - name_mapping = NameMapping() - - grpc_service = TelemetryFrontendService(name_mapping) - grpc_service.start() - - # Wait for Ctrl+C or termination signal - while not terminate.wait(timeout=1.0): pass - - LOGGER.info('Terminating...') - grpc_service.stop() - - LOGGER.info('Bye') - return 0 - -if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file -- GitLab From 10a7540ceab14adb8abad0c751bb3207dcdb90bf Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 8 May 2024 14:20:48 +0000 Subject: [PATCH 106/106] "KafkaProducerService" and "KafkaProducerServiceImpl" class name changed to "TelemetryBackendService" and "TelemetryBackendServiceImpl" respectivily --- ...rService.py => TelemetryBackendService.py} | 18 +---- ...Impl.py => TelemetryBackendServiceImpl.py} | 9 +-- src/telemetry/backend/service/__main__.py | 72 +++++++++++++++++++ .../backend/tests/test_kafka_backend.py | 12 ++-- src/telemetry/frontend/service/__main__.py | 3 +- 5 files changed, 82 insertions(+), 32 deletions(-) rename src/telemetry/backend/service/{KafkaProducerService.py => TelemetryBackendService.py} (64%) rename src/telemetry/backend/service/{KafkaProducerServiceImpl.py => TelemetryBackendServiceImpl.py} (96%) create mode 100644 src/telemetry/backend/service/__main__.py diff --git a/src/telemetry/backend/service/KafkaProducerService.py b/src/telemetry/backend/service/TelemetryBackendService.py similarity index 64% rename from src/telemetry/backend/service/KafkaProducerService.py rename to src/telemetry/backend/service/TelemetryBackendService.py index 0a76b2d99..8e6fb243e 100755 --- a/src/telemetry/backend/service/KafkaProducerService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .KafkaProducerServiceImpl import KafkaProducerServiceImpl - -class KafkaProducerService: +class TelemetryBackendService: """ Class to control Kafka producer functionality. """ @@ -35,19 +33,5 @@ class KafkaProducerService: } return create_kafka_configs - def run_producer(self): - """ - Method to create KafkaProducerServiceImpl object and start producer. - """ - # Create NodeExporterProducer object and run start_producer_thread - producer = KafkaProducerServiceImpl(self.bootstrap_servers, self.node_exporter_endpoint, - self.kafka_topic, self.run_duration, self.fetch_interval - ) - # producer.start_producer_thread() # if threading is required - producer.produce_metrics() # if threading is not required -if __name__ == "__main__": - # Create Kafka producer service object and run producer - kafka_controller = KafkaProducerService() - kafka_controller.run_producer() diff --git a/src/telemetry/backend/service/KafkaProducerServiceImpl.py b/src/telemetry/backend/service/TelemetryBackendServiceImpl.py similarity index 96% rename from src/telemetry/backend/service/KafkaProducerServiceImpl.py rename to src/telemetry/backend/service/TelemetryBackendServiceImpl.py index da5513170..ea57f6167 100755 --- a/src/telemetry/backend/service/KafkaProducerServiceImpl.py +++ b/src/telemetry/backend/service/TelemetryBackendServiceImpl.py @@ -27,7 +27,7 @@ from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_m LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('Telemetry', 'TelemetryBackend') -class KafkaProducerServiceImpl: +class TelemetryBackendServiceImpl: """ Class to fetch metrics from Exporter and produce them to Kafka. """ @@ -171,12 +171,5 @@ class KafkaProducerServiceImpl: finally: kafka_producer.flush() # kafka_producer.close() # this command generates ERROR - # --- - def start_producer_thread(self): - """ - Method to start the producer thread. - """ - producer_thread = threading.Thread(target=self.produce_metrics) - producer_thread.start() # ----------- ABOVE: Actual Implementation of Kafka Producer with Node Exporter ----------- \ No newline at end of file diff --git a/src/telemetry/backend/service/__main__.py b/src/telemetry/backend/service/__main__.py new file mode 100644 index 000000000..10c3f76d1 --- /dev/null +++ b/src/telemetry/backend/service/__main__.py @@ -0,0 +1,72 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import signal +import logging +import threading +from prometheus_client import start_http_server +from monitoring.service.NameMapping import NameMapping +from .KafkaProducerService import KafkaProducerService +from common.Settings import ( + get_log_level, + get_metrics_port) + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER + + log_level = get_log_level() + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + LOGGER = logging.getLogger(__name__) + +# ------- will be added later -------------- + # wait_for_environment_variables([ + # get_env_var_name + + + # ]) +# ------- will be added later -------------- + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting Telemetry Backend...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + + grpc_service = KafkaProducerService(name_mapping) + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/src/telemetry/backend/tests/test_kafka_backend.py b/src/telemetry/backend/tests/test_kafka_backend.py index 05174da2b..ac49bc30f 100644 --- a/src/telemetry/backend/tests/test_kafka_backend.py +++ b/src/telemetry/backend/tests/test_kafka_backend.py @@ -18,8 +18,8 @@ import logging from typing import Tuple from common.proto.context_pb2 import Empty from src.telemetry.frontend.tests.Messages import create_collector_request, create_collector_id -from src.telemetry.backend.service.KafkaProducerService import KafkaProducerService -from src.telemetry.backend.service.KafkaProducerServiceImpl import KafkaProducerServiceImpl +from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService +from src.telemetry.backend.service.TelemetryBackendServiceImpl import TelemetryBackendServiceImpl LOGGER = logging.getLogger(__name__) @@ -29,14 +29,14 @@ LOGGER = logging.getLogger(__name__) ########################### def test_get_kafka_configs(): LOGGER.warning('test_get_kafka_configs requesting') - KafkaProducerServiceObj = KafkaProducerService() + KafkaProducerServiceObj = TelemetryBackendService() response = KafkaProducerServiceObj.generate_kafka_configs() LOGGER.debug(str(response)) assert isinstance(response, dict) def test_export_collector_value(): LOGGER.warning('test_export_collector_value requesting') - response = KafkaProducerServiceImpl.export_collector_value( + response = TelemetryBackendServiceImpl.export_collector_value( create_collector_request('1') ) LOGGER.debug(str(response)) @@ -44,7 +44,7 @@ def test_export_collector_value(): def test_write_to_kafka(): LOGGER.warning('test_write_to_kafka requesting') - collector_value = KafkaProducerServiceImpl.export_collector_value(create_collector_request('1')) - response = KafkaProducerServiceImpl.write_to_kafka(collector_value) # type: ignore (don't know why warning here) + collector_value = TelemetryBackendServiceImpl.export_collector_value(create_collector_request('1')) + response = TelemetryBackendServiceImpl.write_to_kafka(collector_value) # type: ignore (don't know why warning here) LOGGER.debug(str(response)) assert isinstance(response, Empty) diff --git a/src/telemetry/frontend/service/__main__.py b/src/telemetry/frontend/service/__main__.py index afc381e09..0f48a4de1 100644 --- a/src/telemetry/frontend/service/__main__.py +++ b/src/telemetry/frontend/service/__main__.py @@ -15,8 +15,9 @@ import signal import sys import logging, threading -from .TelemetryFrontendService import TelemetryFrontendService +from prometheus_client import start_http_server from monitoring.service.NameMapping import NameMapping +from .TelemetryFrontendService import TelemetryFrontendService from monitoring.service.EventTools import EventsDeviceCollector from common.Settings import ( get_log_level, wait_for_environment_variables, get_env_var_name, -- GitLab