From 4802e99eb328356c125fa15637355963291a4b2b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 10:30:22 +0000 Subject: [PATCH 1/9] Pre-release code fixing and cleanup --- src/context/service/database/Constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index 2ca49338a..79970eacd 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -122,7 +122,7 @@ def upsert_constraints( stmt = delete(klass) if service_uuid is not None: stmt = stmt.where(klass.service_uuid == service_uuid) if slice_uuid is not None: stmt = stmt.where(klass.slice_uuid == slice_uuid ) - stmt = stmt.where(klass.constraint_uuid.in_(uuids_to_delete) + stmt = stmt.where(klass.constraint_uuid.in_(uuids_to_delete)) #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt))) constraint_deletes = session.execute(stmt) -- GitLab From a28cd049770f3b9b7b1d896565dfebb2d3870ac5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 10:57:21 +0000 Subject: [PATCH 2/9] Pre-release fixing --- src/device/requirements.in | 2 +- src/monitoring/requirements.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/device/requirements.in b/src/device/requirements.in index 825ef8e21..869f33f8f 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -14,7 +14,7 @@ anytree==2.8.0 -APScheduler==3.8.1 +APScheduler==3.10.1 cryptography==36.0.2 #fastcache==1.1.0 Jinja2==3.0.3 diff --git a/src/monitoring/requirements.in b/src/monitoring/requirements.in index 981b4cdf2..b7d61021a 100644 --- a/src/monitoring/requirements.in +++ b/src/monitoring/requirements.in @@ -13,7 +13,7 @@ # limitations under the License. anytree==2.8.0 -APScheduler==3.8.1 +APScheduler==3.10.1 #fastcache==1.1.0 #google-api-core #opencensus[stackdriver] -- GitLab From bf2ff0590d2de2a804916b8ba1d53972d750fe28 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 11:54:08 +0000 Subject: [PATCH 3/9] Pre-release fixing --- src/device/requirements.in | 2 +- src/load_generator/requirements.in | 2 +- src/monitoring/requirements.in | 2 +- src/monitoring/tests/test_unitary.py | 9 ++++++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/device/requirements.in b/src/device/requirements.in index 869f33f8f..c81e81460 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -22,7 +22,7 @@ ncclient==0.6.13 p4runtime==1.3.0 paramiko==2.9.2 python-json-logger==2.0.2 -pytz==2021.3 +#pytz==2021.3 #redis==4.1.2 requests==2.27.1 requests-mock==1.9.3 diff --git a/src/load_generator/requirements.in b/src/load_generator/requirements.in index 03a61d7a3..44bd0ef1a 100644 --- a/src/load_generator/requirements.in +++ b/src/load_generator/requirements.in @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -APScheduler==3.8.1 +APScheduler==3.10.1 diff --git a/src/monitoring/requirements.in b/src/monitoring/requirements.in index b7d61021a..4e57dd019 100644 --- a/src/monitoring/requirements.in +++ b/src/monitoring/requirements.in @@ -26,7 +26,7 @@ APScheduler==3.10.1 influx-line-protocol==0.1.4 python-dateutil==2.8.2 python-json-logger==2.0.2 -pytz==2021.3 +#pytz==2021.3 #redis==4.1.2 requests==2.27.1 xmltodict==0.12.0 diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index 4e84431a5..5081df9dd 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -31,6 +31,7 @@ from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.tools.object_factory.ConfigRule import json_config_rule_set from common.tools.service.GenericGrpcService import GenericGrpcService from common.tools.timestamp.Converters import timestamp_utcnow_to_float #, timestamp_string_to_float from context.client.ContextClient import ContextClient @@ -48,7 +49,7 @@ from monitoring.service.NameMapping import NameMapping #from monitoring.service.SubscriptionManager import SubscriptionManager from monitoring.tests.Messages import create_kpi_request, create_kpi_request_d, include_kpi_request, monitor_kpi_request, \ create_kpi_request_c, kpi_query, subs_descriptor, alarm_descriptor, alarm_subscription #, create_kpi_request_b -from monitoring.tests.Objects import DEVICE_DEV1, DEVICE_DEV1_CONNECT_RULES, DEVICE_DEV1_UUID +from monitoring.tests.Objects import DEVICE_DEV1, DEVICE_DEV1_CONNECT_RULES, DEVICE_DEV1_UUID, ENDPOINT_END1_UUID os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' from device.service.drivers import DRIVERS # pylint: disable=wrong-import-position,ungrouped-imports @@ -601,6 +602,12 @@ def test_listen_events( LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID)) device_with_connect_rules = copy.deepcopy(DEVICE_DEV1) device_with_connect_rules['device_config']['config_rules'].extend(DEVICE_DEV1_CONNECT_RULES) + device_with_connect_rules['device_config']['config_rules'].append( + json_config_rule_set( + '/interface[{:s}]'.format(ENDPOINT_END1_UUID), + {'name': ENDPOINT_END1_UUID, 'enabled': True} + ) + ) response = device_client.AddDevice(Device(**device_with_connect_rules)) assert response.device_uuid.uuid == DEVICE_DEV1_UUID -- GitLab From 0223e1068354e22bff2ccd84e53b45d49823eb27 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 12:04:49 +0000 Subject: [PATCH 4/9] Pre-release fixing --- src/monitoring/tests/test_unitary.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index 5081df9dd..ff19e231e 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -13,7 +13,7 @@ # limitations under the License. import copy, os, pytest #, threading, time -import logging +import logging, json #from queue import Queue from random import random from time import sleep @@ -25,7 +25,7 @@ from grpc._channel import _MultiThreadedRendezvous from common.Constants import ServiceNameEnum from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) -from common.proto.context_pb2 import DeviceOperationalStatusEnum, EventTypeEnum, DeviceEvent, Device, Empty +from common.proto.context_pb2 import ConfigActionEnum, DeviceOperationalStatusEnum, EventTypeEnum, DeviceEvent, Device, Empty from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ @@ -602,21 +602,18 @@ def test_listen_events( LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID)) device_with_connect_rules = copy.deepcopy(DEVICE_DEV1) device_with_connect_rules['device_config']['config_rules'].extend(DEVICE_DEV1_CONNECT_RULES) - device_with_connect_rules['device_config']['config_rules'].append( - json_config_rule_set( - '/interface[{:s}]'.format(ENDPOINT_END1_UUID), - {'name': ENDPOINT_END1_UUID, 'enabled': True} - ) - ) response = device_client.AddDevice(Device(**device_with_connect_rules)) assert response.device_uuid.uuid == DEVICE_DEV1_UUID LOGGER.info('Activating Device {:s}'.format(DEVICE_DEV1_UUID)) - device = context_client.GetDevice(response) device_with_op_state = Device() device_with_op_state.CopyFrom(device) device_with_op_state.device_operational_status = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + config_rule = device_with_op_state.device_config.config_rules.add() + config_rule.action = ConfigActionEnum.CONFIGACTION_SET + config_rule.custom.resource_key = '/interface[{:s}]'.format(ENDPOINT_END1_UUID) + config_rule.custom.resource_value = json.dumps({'name': ENDPOINT_END1_UUID, 'enabled': True}) response = context_client.SetDevice(device_with_op_state) assert response.device_uuid.uuid == DEVICE_DEV1_UUID -- GitLab From 2dac882582591385259ac3d91467754cba6bd30f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 12:23:00 +0000 Subject: [PATCH 5/9] Pre-release fixing --- src/monitoring/service/EventTools.py | 50 +++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index be3fe9b92..4dea8221b 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -18,6 +18,7 @@ from common.method_wrappers.ServiceExceptions import ServiceException from common.proto import monitoring_pb2 from common.proto.context_pb2 import ConfigActionEnum, DeviceOperationalStatusEnum, Empty, EventTypeEnum from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.grpc.Tools import grpc_message_to_json_string from context.client.ContextClient import ContextClient from monitoring.client.MonitoringClient import MonitoringClient from monitoring.service.MonitoringServiceServicerImpl import LOGGER @@ -86,10 +87,12 @@ class EventsDeviceCollector: event_type = event.event.event_type device_uuid = event.device_id.device_uuid.uuid if event_type in {EventTypeEnum.EVENTTYPE_REMOVE}: + LOGGER.info('removing device') self._device_to_state.pop(device_uuid, None) continue if event_type not in {EventTypeEnum.EVENTTYPE_CREATE, EventTypeEnum.EVENTTYPE_UPDATE}: + LOGGER.info('unexpected event') # Unknown event type continue @@ -97,37 +100,68 @@ class EventsDeviceCollector: self._name_mapping.set_device_name(device_uuid, device.name) old_operational_status = self._device_to_state.get(device_uuid, DEVICE_OP_STATUS_UNDEFINED) + LOGGER.info('old_operational_status={:s}'.format(str(old_operational_status))) device_was_not_enabled = (old_operational_status in DEVICE_OP_STATUS_NOT_ENABLED) + LOGGER.info('device_was_not_enabled={:s}'.format(str(device_was_not_enabled))) new_operational_status = device.device_operational_status + LOGGER.info('new_operational_status={:s}'.format(str(new_operational_status))) device_is_enabled = (new_operational_status == DEVICE_OP_STATUS_ENABLED) + LOGGER.info('device_is_enabled={:s}'.format(str(device_is_enabled))) self._device_to_state[device_uuid] = new_operational_status activate_monitoring = device_was_not_enabled and device_is_enabled + LOGGER.info('activate_monitoring={:s}'.format(str(activate_monitoring))) if not activate_monitoring: + LOGGER.info('device not ready for activation') # device is not ready for monitoring continue + LOGGER.info('checking endpoints') + enabled_endpoint_names = set() for config_rule in device.device_config.config_rules: - if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue - if config_rule.WhichOneof('config_rule') != 'custom': continue + LOGGER.info(grpc_message_to_json_string(config_rule)) + if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: + LOGGER.info(' not set config rule') + continue + if config_rule.WhichOneof('config_rule') != 'custom': + LOGGER.info(' not custom') + continue str_resource_key = str(config_rule.custom.resource_key) - if not str_resource_key.startswith('/interface['): continue + LOGGER.info(' str_resource_key={:s}'.format(str(str_resource_key))) + if not str_resource_key.startswith('/interface['): + LOGGER.info(' not /interface') + continue json_resource_value = json.loads(config_rule.custom.resource_value) - if 'name' not in json_resource_value: continue - if 'enabled' not in json_resource_value: continue - if not json_resource_value['enabled']: continue + LOGGER.info(' json_resource_value={:s}'.format(str(json_resource_value))) + if 'name' not in json_resource_value: + LOGGER.info(' missing name') + continue + if 'enabled' not in json_resource_value: + LOGGER.info(' missing enabled') + continue + if not json_resource_value['enabled']: + LOGGER.info(' not enabled') + continue + LOGGER.info(' enabled') enabled_endpoint_names.add(json_resource_value['name']) + LOGGER.info('enabled_endpoint_names={:s}'.format(str(enabled_endpoint_names))) + for endpoint in device.device_endpoints: - if endpoint.name not in enabled_endpoint_names: continue + LOGGER.info(' endpoint: {:s}'.format(str(endpoint.name))) + if endpoint.name not in enabled_endpoint_names: + LOGGER.info(' skipping not enabled') + continue endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid self._name_mapping.set_endpoint_name(endpoint_uuid, endpoint.name) for value in endpoint.kpi_sample_types: - if value == KPISAMPLETYPE_UNKNOWN: continue + if value == KPISAMPLETYPE_UNKNOWN: + LOGGER.info(' unknown kpi sample type') + continue kpi_descriptor = monitoring_pb2.KpiDescriptor() kpi_descriptor.kpi_description = device.device_type -- GitLab From 740d4f6c38da33b37585c104fdd5f93136886922 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 13:02:36 +0000 Subject: [PATCH 6/9] Pre-release fixing --- src/monitoring/service/EventTools.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 4dea8221b..bb220e364 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -100,20 +100,14 @@ class EventsDeviceCollector: self._name_mapping.set_device_name(device_uuid, device.name) old_operational_status = self._device_to_state.get(device_uuid, DEVICE_OP_STATUS_UNDEFINED) - LOGGER.info('old_operational_status={:s}'.format(str(old_operational_status))) device_was_not_enabled = (old_operational_status in DEVICE_OP_STATUS_NOT_ENABLED) - LOGGER.info('device_was_not_enabled={:s}'.format(str(device_was_not_enabled))) new_operational_status = device.device_operational_status - LOGGER.info('new_operational_status={:s}'.format(str(new_operational_status))) device_is_enabled = (new_operational_status == DEVICE_OP_STATUS_ENABLED) - LOGGER.info('device_is_enabled={:s}'.format(str(device_is_enabled))) self._device_to_state[device_uuid] = new_operational_status activate_monitoring = device_was_not_enabled and device_is_enabled - LOGGER.info('activate_monitoring={:s}'.format(str(activate_monitoring))) if not activate_monitoring: - LOGGER.info('device not ready for activation') # device is not ready for monitoring continue @@ -122,17 +116,11 @@ class EventsDeviceCollector: enabled_endpoint_names = set() for config_rule in device.device_config.config_rules: LOGGER.info(grpc_message_to_json_string(config_rule)) - if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: - LOGGER.info(' not set config rule') - continue - if config_rule.WhichOneof('config_rule') != 'custom': - LOGGER.info(' not custom') - continue + if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue + if config_rule.WhichOneof('config_rule') != 'custom': continue str_resource_key = str(config_rule.custom.resource_key) LOGGER.info(' str_resource_key={:s}'.format(str(str_resource_key))) - if not str_resource_key.startswith('/interface['): - LOGGER.info(' not /interface') - continue + if not str_resource_key.startswith('/interface['): continue json_resource_value = json.loads(config_rule.custom.resource_value) LOGGER.info(' json_resource_value={:s}'.format(str(json_resource_value))) if 'name' not in json_resource_value: @@ -150,12 +138,15 @@ class EventsDeviceCollector: LOGGER.info('enabled_endpoint_names={:s}'.format(str(enabled_endpoint_names))) for endpoint in device.device_endpoints: - LOGGER.info(' endpoint: {:s}'.format(str(endpoint.name))) - if endpoint.name not in enabled_endpoint_names: + endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid + endpoint_name_or_uuid = endpoint.name + if endpoint_name_or_uuid is None or len(endpoint_name_or_uuid) == 0: + endpoint_name_or_uuid = endpoint_uuid + LOGGER.info(' endpoint: {:s}'.format(str(endpoint_name_or_uuid))) + if endpoint_name_or_uuid not in enabled_endpoint_names: LOGGER.info(' skipping not enabled') continue - endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid self._name_mapping.set_endpoint_name(endpoint_uuid, endpoint.name) for value in endpoint.kpi_sample_types: -- GitLab From 6f610f6caf15726e11f44e26aacde45416c77ba1 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 15:23:52 +0000 Subject: [PATCH 7/9] Pre-release fixing --- src/monitoring/service/EventTools.py | 29 +++++----------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index bb220e364..c1cd4d006 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -87,12 +87,10 @@ class EventsDeviceCollector: event_type = event.event.event_type device_uuid = event.device_id.device_uuid.uuid if event_type in {EventTypeEnum.EVENTTYPE_REMOVE}: - LOGGER.info('removing device') self._device_to_state.pop(device_uuid, None) continue if event_type not in {EventTypeEnum.EVENTTYPE_CREATE, EventTypeEnum.EVENTTYPE_UPDATE}: - LOGGER.info('unexpected event') # Unknown event type continue @@ -111,7 +109,6 @@ class EventsDeviceCollector: # device is not ready for monitoring continue - LOGGER.info('checking endpoints') enabled_endpoint_names = set() for config_rule in device.device_config.config_rules: @@ -119,40 +116,24 @@ class EventsDeviceCollector: if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue if config_rule.WhichOneof('config_rule') != 'custom': continue str_resource_key = str(config_rule.custom.resource_key) - LOGGER.info(' str_resource_key={:s}'.format(str(str_resource_key))) if not str_resource_key.startswith('/interface['): continue json_resource_value = json.loads(config_rule.custom.resource_value) - LOGGER.info(' json_resource_value={:s}'.format(str(json_resource_value))) - if 'name' not in json_resource_value: - LOGGER.info(' missing name') - continue - if 'enabled' not in json_resource_value: - LOGGER.info(' missing enabled') - continue - if not json_resource_value['enabled']: - LOGGER.info(' not enabled') - continue - LOGGER.info(' enabled') + if 'name' not in json_resource_value: continue + if 'enabled' not in json_resource_value: continue + if not json_resource_value['enabled']: continue enabled_endpoint_names.add(json_resource_value['name']) - LOGGER.info('enabled_endpoint_names={:s}'.format(str(enabled_endpoint_names))) - for endpoint in device.device_endpoints: endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid endpoint_name_or_uuid = endpoint.name if endpoint_name_or_uuid is None or len(endpoint_name_or_uuid) == 0: endpoint_name_or_uuid = endpoint_uuid - LOGGER.info(' endpoint: {:s}'.format(str(endpoint_name_or_uuid))) - if endpoint_name_or_uuid not in enabled_endpoint_names: - LOGGER.info(' skipping not enabled') - continue + if endpoint_name_or_uuid not in enabled_endpoint_names: continue self._name_mapping.set_endpoint_name(endpoint_uuid, endpoint.name) for value in endpoint.kpi_sample_types: - if value == KPISAMPLETYPE_UNKNOWN: - LOGGER.info(' unknown kpi sample type') - continue + if value == KPISAMPLETYPE_UNKNOWN: continue kpi_descriptor = monitoring_pb2.KpiDescriptor() kpi_descriptor.kpi_description = device.device_type -- GitLab From 4f3f496294fef8be840c779efedc857154677974 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 15:31:43 +0000 Subject: [PATCH 8/9] Pre-release fixing --- src/monitoring/service/EventTools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index c1cd4d006..667169c93 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -18,7 +18,6 @@ from common.method_wrappers.ServiceExceptions import ServiceException from common.proto import monitoring_pb2 from common.proto.context_pb2 import ConfigActionEnum, DeviceOperationalStatusEnum, Empty, EventTypeEnum from common.proto.kpi_sample_types_pb2 import KpiSampleType -from common.tools.grpc.Tools import grpc_message_to_json_string from context.client.ContextClient import ContextClient from monitoring.client.MonitoringClient import MonitoringClient from monitoring.service.MonitoringServiceServicerImpl import LOGGER @@ -112,7 +111,6 @@ class EventsDeviceCollector: enabled_endpoint_names = set() for config_rule in device.device_config.config_rules: - LOGGER.info(grpc_message_to_json_string(config_rule)) if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue if config_rule.WhichOneof('config_rule') != 'custom': continue str_resource_key = str(config_rule.custom.resource_key) -- GitLab From ddd06652086bd609fe72d24494eaa8f23c7f78fd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 10 Jul 2023 15:34:20 +0000 Subject: [PATCH 9/9] Pre-release fixing --- src/monitoring/service/EventTools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 667169c93..a840cde45 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -108,7 +108,6 @@ class EventsDeviceCollector: # device is not ready for monitoring continue - enabled_endpoint_names = set() for config_rule in device.device_config.config_rules: if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue -- GitLab