From 95f27fa082e4660025790c22be467b7c27081d00 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 12:35:12 +0000 Subject: [PATCH 01/31] Common MessageBroker - NATS: - added support for username and password - added support for direct URI --- .../backend/nats/NatsBackend.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/common/message_broker/backend/nats/NatsBackend.py b/src/common/message_broker/backend/nats/NatsBackend.py index 6c644a0a8..4a6d5e9cd 100644 --- a/src/common/message_broker/backend/nats/NatsBackend.py +++ b/src/common/message_broker/backend/nats/NatsBackend.py @@ -19,18 +19,24 @@ from common.message_broker.Message import Message from .._Backend import _Backend from .NatsBackendThread import NatsBackendThread -DEFAULT_NATS_URI = 'nats://127.0.0.1:4222' -#NATS_URI_TEMPLATE = 'nats://{:s}:{:s}@nats.{:s}.svc.cluster.local:{:s}' # with authentication -NATS_URI_TEMPLATE = 'nats://nats.{:s}.svc.cluster.local:{:s}' +NATS_URI_TEMPLATE_AUTH = 'nats://{:s}:{:s}@nats.{:s}.svc.cluster.local:{:s}' +NATS_URI_TEMPLATE_NOAUTH = 'nats://nats.{:s}.svc.cluster.local:{:s}' class NatsBackend(_Backend): def __init__(self, **settings) -> None: # pylint: disable=super-init-not-called - nats_namespace = get_setting('NATS_NAMESPACE', settings=settings) - nats_client_port = get_setting('NATS_CLIENT_PORT', settings=settings) - if nats_namespace is None or nats_client_port is None: - nats_uri = get_setting('NATS_URI', settings=settings, default=DEFAULT_NATS_URI) - else: - nats_uri = NATS_URI_TEMPLATE.format(nats_namespace, nats_client_port) + nats_uri = get_setting('NATS_URI', settings=settings, default=None) + if nats_uri is None: + nats_namespace = get_setting('NATS_NAMESPACE', settings=settings) + nats_client_port = get_setting('NATS_CLIENT_PORT', settings=settings) + nats_username = get_setting('NATS_USERNAME', settings=settings, default=None) + nats_password = get_setting('NATS_PASSWORD', settings=settings, default=None) + if nats_username is None or nats_password is None: + nats_uri = NATS_URI_TEMPLATE_NOAUTH.format( + nats_namespace, nats_client_port) + else: + nats_uri = NATS_URI_TEMPLATE_AUTH.format( + nats_username, nats_password, nats_namespace, nats_client_port) + self._terminate = threading.Event() self._nats_backend_thread = NatsBackendThread(nats_uri) self._nats_backend_thread.start() -- GitLab From 227bdc14cbb7f519f446d1680d0e03c1454ac875 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 12:35:32 +0000 Subject: [PATCH 02/31] Context component: - added support for direct CockroachDB URI --- src/context/service/database/Engine.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/context/service/database/Engine.py b/src/context/service/database/Engine.py index a37ec0c1e..46c1b8c25 100644 --- a/src/context/service/database/Engine.py +++ b/src/context/service/database/Engine.py @@ -24,15 +24,16 @@ CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster class Engine: @staticmethod def get_engine() -> sqlalchemy.engine.Engine: - CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') - CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') - CRDB_DATABASE = get_setting('CRDB_DATABASE') - CRDB_USERNAME = get_setting('CRDB_USERNAME') - CRDB_PASSWORD = get_setting('CRDB_PASSWORD') - CRDB_SSLMODE = get_setting('CRDB_SSLMODE') - - crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = get_setting('CRDB_DATABASE') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine( -- GitLab From 87d0e798d170c08e2986eae361fcbbf448bfee5f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 12:44:44 +0000 Subject: [PATCH 03/31] Device component - TAPI Driver: - added basic support for HTTPS - added optional support for authentication --- .../service/drivers/transport_api/Tools.py | 21 ++++++++++++------- .../transport_api/TransportApiDriver.py | 19 +++++++++++------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/device/service/drivers/transport_api/Tools.py b/src/device/service/drivers/transport_api/Tools.py index 898929419..6b1be361b 100644 --- a/src/device/service/drivers/transport_api/Tools.py +++ b/src/device/service/drivers/transport_api/Tools.py @@ -13,6 +13,8 @@ # limitations under the License. import json, logging, requests +from requests.auth import HTTPBasicAuth +from typing import Optional from device.service.driver_api._Driver import RESOURCE_ENDPOINTS LOGGER = logging.getLogger(__name__) @@ -28,11 +30,13 @@ def find_key(resource, key): return json.loads(resource[1])[key] -def config_getter(root_url, resource_key, timeout): +def config_getter( + root_url : str, resource_key : str, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None +): url = '{:s}/restconf/data/tapi-common:context'.format(root_url) result = [] try: - response = requests.get(url, timeout=timeout) + response = requests.get(url, timeout=timeout, verify=False, auth=auth) except requests.exceptions.Timeout: LOGGER.exception('Timeout connecting {:s}'.format(url)) except Exception as e: # pylint: disable=broad-except @@ -63,8 +67,10 @@ def config_getter(root_url, resource_key, timeout): return result def create_connectivity_service( - root_url, timeout, uuid, input_sip, output_sip, direction, capacity_value, capacity_unit, layer_protocol_name, - layer_protocol_qualifier): + root_url, uuid, input_sip, output_sip, direction, capacity_value, capacity_unit, layer_protocol_name, + layer_protocol_qualifier, + auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None +): url = '{:s}/restconf/data/tapi-common:context/tapi-connectivity:connectivity-context'.format(root_url) headers = {'content-type': 'application/json'} @@ -105,7 +111,8 @@ def create_connectivity_service( results = [] try: LOGGER.info('Connectivity service {:s}: {:s}'.format(str(uuid), str(data))) - response = requests.post(url=url, data=json.dumps(data), timeout=timeout, headers=headers) + response = requests.post( + url=url, data=json.dumps(data), timeout=timeout, headers=headers, verify=False, auth=auth) LOGGER.info('TAPI response: {:s}'.format(str(response))) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception creating ConnectivityService(uuid={:s}, data={:s})'.format(str(uuid), str(data))) @@ -117,12 +124,12 @@ def create_connectivity_service( results.append(response.status_code in HTTP_OK_CODES) return results -def delete_connectivity_service(root_url, timeout, uuid): +def delete_connectivity_service(root_url, uuid, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None): url = '{:s}/restconf/data/tapi-common:context/tapi-connectivity:connectivity-context/connectivity-service={:s}' url = url.format(root_url, uuid) results = [] try: - response = requests.delete(url=url, timeout=timeout) + response = requests.delete(url=url, timeout=timeout, verify=False, auth=auth) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception deleting ConnectivityService(uuid={:s})'.format(str(uuid))) results.append(e) diff --git a/src/device/service/drivers/transport_api/TransportApiDriver.py b/src/device/service/drivers/transport_api/TransportApiDriver.py index 71d7aa336..874d4aeb6 100644 --- a/src/device/service/drivers/transport_api/TransportApiDriver.py +++ b/src/device/service/drivers/transport_api/TransportApiDriver.py @@ -13,6 +13,7 @@ # limitations under the License. import logging, requests, threading +from requests.auth import HTTPBasicAuth from typing import Any, Iterator, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.type_checkers.Checkers import chk_string, chk_type @@ -29,7 +30,11 @@ class TransportApiDriver(_Driver): self.__lock = threading.Lock() self.__started = threading.Event() self.__terminate = threading.Event() - self.__tapi_root = 'http://' + address + ':' + str(port) + username = settings.get('username') + password = settings.get('password') + self.__auth = HTTPBasicAuth(username, password) if username is not None and password is not None else None + scheme = settings.get('scheme', 'http') + self.__tapi_root = '{:s}://{:s}:{:d}'.format(scheme, address, int(port)) self.__timeout = int(settings.get('timeout', 120)) def Connect(self) -> bool: @@ -37,7 +42,7 @@ class TransportApiDriver(_Driver): with self.__lock: if self.__started.is_set(): return True try: - requests.get(url, timeout=self.__timeout) + requests.get(url, timeout=self.__timeout, verify=False, auth=self.__auth) except requests.exceptions.Timeout: LOGGER.exception('Timeout connecting {:s}'.format(str(self.__tapi_root))) return False @@ -67,7 +72,8 @@ class TransportApiDriver(_Driver): for i, resource_key in enumerate(resource_keys): str_resource_name = 'resource_key[#{:d}]'.format(i) chk_string(str_resource_name, resource_key, allow_empty=False) - results.extend(config_getter(self.__tapi_root, resource_key, self.__timeout)) + results.extend(config_getter( + self.__tapi_root, resource_key, timeout=self.__timeout, auth=self.__auth)) return results @metered_subclass_method(METRICS_POOL) @@ -89,8 +95,8 @@ class TransportApiDriver(_Driver): direction = find_key(resource, 'direction') data = create_connectivity_service( - self.__tapi_root, self.__timeout, uuid, input_sip, output_sip, direction, capacity_value, - capacity_unit, layer_protocol_name, layer_protocol_qualifier) + self.__tapi_root, uuid, input_sip, output_sip, direction, capacity_value, capacity_unit, + layer_protocol_name, layer_protocol_qualifier, timeout=self.__timeout, auth=self.__auth) results.extend(data) return results @@ -102,7 +108,8 @@ class TransportApiDriver(_Driver): for resource in resources: LOGGER.info('resource = {:s}'.format(str(resource))) uuid = find_key(resource, 'uuid') - results.extend(delete_connectivity_service(self.__tapi_root, self.__timeout, uuid)) + results.extend(delete_connectivity_service( + self.__tapi_root, uuid, timeout=self.__timeout, auth=self.__auth)) return results @metered_subclass_method(METRICS_POOL) -- GitLab From 2887024257d5f06cd86a5e046fe5612cde9751de Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 14:48:05 +0000 Subject: [PATCH 04/31] Deploy CockroachDB: - moved Mgmt WebUI to port 8081 to prevent collisions with TAPI server used in hackfest --- deploy/crdb.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 76aa07370..598980ac8 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -111,7 +111,7 @@ function crdb_deploy_single() { echo echo "CockroachDB Port Mapping" - echo ">>> Expose CockroachDB SQL port (26257)" + echo ">>> Expose CockroachDB SQL port (26257->26257)" CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') PATCH='{"data": {"'${CRDB_SQL_PORT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_SQL_PORT}'"}}' kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" @@ -122,12 +122,13 @@ function crdb_deploy_single() { kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" echo - echo ">>> Expose CockroachDB HTTP Mgmt GUI port (8080)" + echo ">>> Expose CockroachDB HTTP Mgmt GUI port (8080->8081)" + CRDB_GUI_PORT_EXT="8081" CRDB_GUI_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') - PATCH='{"data": {"'${CRDB_GUI_PORT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_GUI_PORT}'"}}' + PATCH='{"data": {"'${CRDB_GUI_PORT_EXT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_GUI_PORT}'"}}' kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" - PORT_MAP='{"containerPort": '${CRDB_GUI_PORT}', "hostPort": '${CRDB_GUI_PORT}'}' + PORT_MAP='{"containerPort": '${CRDB_GUI_PORT_EXT}', "hostPort": '${CRDB_GUI_PORT_EXT}'}' CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" @@ -275,12 +276,13 @@ function crdb_deploy_cluster() { kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" echo - echo ">>> Expose CockroachDB HTTP Mgmt GUI port (8080)" + echo ">>> Expose CockroachDB HTTP Mgmt GUI port (8080->8081)" + CRDB_GUI_PORT_EXT="8081" CRDB_GUI_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}') - PATCH='{"data": {"'${CRDB_GUI_PORT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_GUI_PORT}'"}}' + PATCH='{"data": {"'${CRDB_GUI_PORT_EXT}'": "'${CRDB_NAMESPACE}'/cockroachdb-public:'${CRDB_GUI_PORT}'"}}' kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" - PORT_MAP='{"containerPort": '${CRDB_GUI_PORT}', "hostPort": '${CRDB_GUI_PORT}'}' + PORT_MAP='{"containerPort": '${CRDB_GUI_PORT_EXT}', "hostPort": '${CRDB_GUI_PORT_EXT}'}' CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" -- GitLab From 6103d18dcb6efd26ef84f512a59bdfce65b263c5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 14:48:29 +0000 Subject: [PATCH 05/31] Commo method-wrappers: - added missing "raise" statement when something goes wrong --- src/common/method_wrappers/Decorator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/method_wrappers/Decorator.py b/src/common/method_wrappers/Decorator.py index f918b8458..558e5c17a 100644 --- a/src/common/method_wrappers/Decorator.py +++ b/src/common/method_wrappers/Decorator.py @@ -169,6 +169,7 @@ def metered_subclass_method(metrics_pool : MetricsPool): raise except Exception: # pylint: disable=broad-except counter_failed.inc() + raise return inner_wrapper return outer_wrapper -- GitLab From d8aae80fab3d569831785613fc89dc24f20167be Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 14:49:13 +0000 Subject: [PATCH 06/31] Device component - TAPI Driver: - corrected parsing of endpoints to improve flexibility in retrieved fields --- .../service/drivers/transport_api/Tools.py | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/device/service/drivers/transport_api/Tools.py b/src/device/service/drivers/transport_api/Tools.py index 6b1be361b..e54e6497a 100644 --- a/src/device/service/drivers/transport_api/Tools.py +++ b/src/device/service/drivers/transport_api/Tools.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging, requests +import json, logging, operator, requests from requests.auth import HTTPBasicAuth from typing import Optional from device.service.driver_api._Driver import RESOURCE_ENDPOINTS @@ -39,30 +39,40 @@ def config_getter( response = requests.get(url, timeout=timeout, verify=False, auth=auth) except requests.exceptions.Timeout: LOGGER.exception('Timeout connecting {:s}'.format(url)) + return result except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception retrieving {:s}'.format(resource_key)) result.append((resource_key, e)) - else: + return result + + try: context = json.loads(response.content) + except Exception as e: # pylint: disable=broad-except + LOGGER.warning('Unable to decode reply: {:s}'.format(str(response.content))) + result.append((resource_key, e)) + return result + + if resource_key != RESOURCE_ENDPOINTS: return result + + if 'tapi-common:context' in context: + context = context['tapi-common:context'] + elif 'context' in context: + context = context['context'] - if resource_key == RESOURCE_ENDPOINTS: - if 'tapi-common:context' in context: - context = context['tapi-common:context'] - elif 'context' in context: - context = context['context'] - for sip in context['service-interface-point']: - layer_protocol_name = sip.get('layer-protocol-name', '?') - supportable_spectrum = sip.get('tapi-photonic-media:media-channel-service-interface-point-spec', {}) - supportable_spectrum = supportable_spectrum.get('mc-pool', {}) - supportable_spectrum = supportable_spectrum.get('supportable-spectrum', []) - supportable_spectrum = supportable_spectrum[0] if len(supportable_spectrum) == 1 else {} - grid_type = supportable_spectrum.get('frequency-constraint', {}).get('grid-type') - granularity = supportable_spectrum.get('frequency-constraint', {}).get('adjustment-granularity') - direction = sip.get('direction', '?') - endpoint_type = ':'.join([layer_protocol_name, grid_type, granularity, direction]) - endpoint_url = '/endpoints/endpoint[{:s}]'.format(sip['uuid']) - endpoint_data = {'uuid': sip['uuid'], 'type': endpoint_type} - result.append((endpoint_url, endpoint_data)) + for sip in context['service-interface-point']: + layer_protocol_name = sip.get('layer-protocol-name', '?') + supportable_spectrum = sip.get('tapi-photonic-media:media-channel-service-interface-point-spec', {}) + supportable_spectrum = supportable_spectrum.get('mc-pool', {}) + supportable_spectrum = supportable_spectrum.get('supportable-spectrum', []) + supportable_spectrum = supportable_spectrum[0] if len(supportable_spectrum) == 1 else {} + grid_type = supportable_spectrum.get('frequency-constraint', {}).get('grid-type') + granularity = supportable_spectrum.get('frequency-constraint', {}).get('adjustment-granularity') + direction = sip.get('direction', '?') + endpoint_type = [layer_protocol_name, grid_type, granularity, direction] + str_endpoint_type = ':'.join(filter(lambda i: operator.is_not(i, None), endpoint_type)) + endpoint_url = '/endpoints/endpoint[{:s}]'.format(sip['uuid']) + endpoint_data = {'uuid': sip['uuid'], 'type': str_endpoint_type} + result.append((endpoint_url, endpoint_data)) return result -- GitLab From 9ccc623ac7713d74b3f79d0592ae6569d471047c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 17:06:04 +0000 Subject: [PATCH 07/31] PathComp component: - Frontend: ensure links have exactly 2 endpoints - Backend: allow larger number of endpoints per node --- src/pathcomp/backend/pathComp_tools.h | 6 +++--- src/pathcomp/frontend/service/algorithms/_Algorithm.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pathcomp/backend/pathComp_tools.h b/src/pathcomp/backend/pathComp_tools.h index adbbf30c4..118781eb0 100644 --- a/src/pathcomp/backend/pathComp_tools.h +++ b/src/pathcomp/backend/pathComp_tools.h @@ -121,7 +121,7 @@ struct map_nodes_t { }; #define MAX_NUM_VERTICES 20 // 100 # LGR: reduced from 100 to 20 to divide by 5 the memory used -#define MAX_NUM_EDGES 20 // 100 # LGR: reduced from 100 to 20 to divide by 5 the memory used +#define MAX_NUM_EDGES 40 // 100 # LGR: reduced from 100 to 40 to divide by 2.5 the memory used // Structures for the graph composition struct targetNodes_t { // remote / targeted node @@ -154,7 +154,7 @@ struct context_t { //////////////////////////////////////////////////// // Structure for the Set of Contexts /////////////////////////////////////////////////// -#define MAX_NUMBER_CONTEXT 5 // 100 # LGR: reduced from 100 to 5 to divide by 20 the memory used +#define MAX_NUMBER_CONTEXT 1 // 100 # LGR: reduced from 100 to 1 to divide by 100 the memory used struct contextSet_t { struct context_t contextList[MAX_NUMBER_CONTEXT]; gint num_context_set; @@ -251,7 +251,7 @@ struct endPoint_t { // Structure for the device contents /////////////////////////////////////////////////////////////////// #define MAX_DEV_TYPE_SIZE 128 -#define MAX_DEV_ENDPOINT_LENGTH 10 +#define MAX_DEV_ENDPOINT_LENGTH 40 // 10 # LGR: controllers might have large number of endpoints struct device_t { gchar deviceId[UUID_CHAR_LENGTH]; // device ID using UUID (128 bits) diff --git a/src/pathcomp/frontend/service/algorithms/_Algorithm.py b/src/pathcomp/frontend/service/algorithms/_Algorithm.py index 5c49a1fec..bf19ed3e1 100644 --- a/src/pathcomp/frontend/service/algorithms/_Algorithm.py +++ b/src/pathcomp/frontend/service/algorithms/_Algorithm.py @@ -68,6 +68,7 @@ class _Algorithm: if isinstance(grpc_links, LinkList): grpc_links = grpc_links.links for grpc_link in grpc_links: json_link = compose_link(grpc_link) + if len(json_link['link_endpoint_ids']) != 2: continue self.link_list.append(json_link) link_uuid = json_link['link_Id'] -- GitLab From 941df77d4728ecd5e0b5f880de823781cb12aa08 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 17:07:34 +0000 Subject: [PATCH 08/31] Service component: - store service in UpdateService to get UUIDs from given endpoint names and store constraints - code cleanup --- .../service/ServiceServiceServicerImpl.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index bf1520270..50f645e42 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -22,9 +22,9 @@ from common.proto.service_pb2_grpc import ServiceServiceServicer from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string from context.client.ContextClient import ContextClient from pathcomp.frontend.client.PathCompClient import PathCompClient -from .tools.ContextGetters import get_service from .service_handler_api.ServiceHandlerFactory import ServiceHandlerFactory from .task_scheduler.TaskScheduler import TasksScheduler +from .tools.ContextGetters import get_service LOGGER = logging.getLogger(__name__) @@ -40,10 +40,6 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): def CreateService(self, request : Service, context : grpc.ServicerContext) -> ServiceId: LOGGER.info('[CreateService] begin ; request = {:s}'.format(grpc_message_to_json_string(request))) - service_id = request.service_id - service_uuid = service_id.service_uuid.uuid - service_context_uuid = service_id.context_id.context_uuid.uuid - if len(request.service_endpoint_ids) > 0: unexpected_endpoints = [] for service_endpoint_id in request.service_endpoint_ids: @@ -97,8 +93,18 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): _service : Optional[Service] = get_service(context_client, request.service_id) service = Service() service.CopyFrom(request if _service is None else _service) - service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED - context_client.SetService(service) + service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED # pylint: disable=no-member + + del service.service_endpoint_ids[:] # pylint: disable=no-member + for service_endpoint_id in request.service_endpoint_ids: + service.service_endpoint_ids.add().CopyFrom(service_endpoint_id) # pylint: disable=no-member + + del service.service_constraints[:] # pylint: disable=no-member + for service_constraint in request.service_constraints: + service.service_constraints.add().CopyFrom(service_constraint) # pylint: disable=no-member + + service_id_with_uuids = context_client.SetService(service) + service_with_uuids = context_client.GetService(service_id_with_uuids) num_disjoint_paths = None for constraint in request.service_constraints: @@ -107,14 +113,14 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): break tasks_scheduler = TasksScheduler(self.service_handler_factory) - if len(request.service_endpoint_ids) >= (2 if num_disjoint_paths is None else 4): + if len(service_with_uuids.service_endpoint_ids) >= (2 if num_disjoint_paths is None else 4): pathcomp_request = PathCompRequest() - pathcomp_request.services.append(request) + pathcomp_request.services.append(service_with_uuids) # pylint: disable=no-member if num_disjoint_paths is None: - pathcomp_request.shortest_path.Clear() + pathcomp_request.shortest_path.Clear() # pylint: disable=no-member else: - pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths + pathcomp_request.k_disjoint_path.num_disjoint = num_disjoint_paths # pylint: disable=no-member LOGGER.info('pathcomp_request={:s}'.format(grpc_message_to_json_string(pathcomp_request))) pathcomp = PathCompClient() @@ -128,7 +134,7 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): tasks_scheduler.compose_from_pathcompreply(pathcomp_reply, is_delete=False) tasks_scheduler.execute_all() - return request.service_id + return service_with_uuids.service_id @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteService(self, request : ServiceId, context : grpc.ServicerContext) -> Empty: @@ -142,6 +148,7 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): if _service is None: raise Exception('Service({:s}) not found'.format(grpc_message_to_json_string(request))) service = Service() service.CopyFrom(_service) + # pylint: disable=no-member service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PENDING_REMOVAL context_client.SetService(service) -- GitLab From 2645c398feef1019f5a0ab2741d7ffe7729b63f4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 17:09:25 +0000 Subject: [PATCH 09/31] WebUI component: - Device List: show number of endpoints instead of detail - Device List: show number of rules - Remove UUIDs from topology elements - Service List: add service name - Service Details: add service name - Slice List: add slice name - Slice Details: add slice name --- src/webui/service/templates/device/home.html | 44 +++++-------------- src/webui/service/templates/js/topology.js | 4 +- .../service/templates/service/detail.html | 3 +- src/webui/service/templates/service/home.html | 6 ++- src/webui/service/templates/slice/detail.html | 3 +- src/webui/service/templates/slice/home.html | 9 ++-- 6 files changed, 25 insertions(+), 44 deletions(-) diff --git a/src/webui/service/templates/device/home.html b/src/webui/service/templates/device/home.html index 7b4437cce..ef4345b36 100644 --- a/src/webui/service/templates/device/home.html +++ b/src/webui/service/templates/device/home.html @@ -48,7 +48,7 @@ Endpoints Drivers Status - + Config Rules @@ -56,40 +56,16 @@ {% if devices %} {% for device in devices %} - - {{ device.device_id.device_uuid.uuid }} - - - {{ device.name }} - - - {{ device.device_type }} - - -
    - {% for end_point in device.device_endpoints %} -
  • {{ end_point.endpoint_id.endpoint_uuid.uuid }}
    • - {% endfor %} -
- - -
    - {% for driver in device.device_drivers %} -
  • {{ dde.Name(driver).replace('DEVICEDRIVER_', '').replace('UNDEFINED', 'EMULATED') }}
    • - {% endfor %} -
- + {{ device.device_id.device_uuid.uuid }} + {{ device.name }} + {{ device.device_type }} + {{ device.device_endpoints | length }} +
    {% for driver in device.device_drivers %} +
  • {{ dde.Name(driver).replace('DEVICEDRIVER_', '').replace('UNDEFINED', 'EMULATED') }}
    • + {% endfor %} +
{{ dose.Name(device.device_operational_status).replace('DEVICEOPERATIONALSTATUS_', '') }} - + {{ device.device_config.config_rules | length }} diff --git a/src/webui/service/templates/js/topology.js b/src/webui/service/templates/js/topology.js index adcabf62c..47c246614 100644 --- a/src/webui/service/templates/js/topology.js +++ b/src/webui/service/templates/js/topology.js @@ -88,9 +88,9 @@ d3.json("{{ url_for('main.topology') }}", function(data) { .call(d3.drag().on("start", dragstarted).on("drag", dragged).on("end", dragended)); // node tooltip - node.append("title").text(function(n) { return n.name + ' (' + n.id + ')'; }); + node.append("title").text(function(n) { return n.name; }); // link tooltip - link.append("title").text(function(l) { return l.name + ' (' + l.id + ')'; }); + link.append("title").text(function(l) { return l.name; }); // link style link diff --git a/src/webui/service/templates/service/detail.html b/src/webui/service/templates/service/detail.html index 67b240b3d..e8d3a4221 100644 --- a/src/webui/service/templates/service/detail.html +++ b/src/webui/service/templates/service/detail.html @@ -17,7 +17,7 @@ {% extends 'base.html' %} {% block content %} -

Service {{ service.service_id.service_uuid.uuid }}

+

Service {{ service.name }} ({{ service.service_id.service_uuid.uuid }})

@@ -45,6 +45,7 @@
Context: {{ service.service_id.context_id.context_uuid.uuid }}
UUID: {{ service.service_id.service_uuid.uuid }}
+ Name: {{ service.name }}
Type: {{ ste.Name(service.service_type).replace('SERVICETYPE_', '') }}
Status: {{ sse.Name(service.service_status.service_status).replace('SERVICESTATUS_', '') }}
diff --git a/src/webui/service/templates/service/home.html b/src/webui/service/templates/service/home.html index c0a01839b..280685fc5 100644 --- a/src/webui/service/templates/service/home.html +++ b/src/webui/service/templates/service/home.html @@ -43,7 +43,8 @@ - + + @@ -59,6 +60,9 @@ {{ service.service_id.service_uuid.uuid }} + diff --git a/src/webui/service/templates/slice/detail.html b/src/webui/service/templates/slice/detail.html index 404dede39..4f26c75a5 100644 --- a/src/webui/service/templates/slice/detail.html +++ b/src/webui/service/templates/slice/detail.html @@ -17,7 +17,7 @@ {% extends 'base.html' %} {% block content %} -

Slice {{ slice.slice_id.slice_uuid.uuid }}

+

Slice {{ slice.name }} ({{ slice.slice_id.slice_uuid.uuid }})

@@ -46,6 +46,7 @@
Context: {{ slice.slice_id.context_id.context_uuid.uuid }}
UUID: {{ slice.slice_id.slice_uuid.uuid }}
+ Name: {{ slice.name }}
Owner: {{ slice.slice_owner.owner_uuid.uuid }}
Status: {{ sse.Name(slice.slice_status.slice_status).replace('SLICESTATUS_', '') }}
diff --git a/src/webui/service/templates/slice/home.html b/src/webui/service/templates/slice/home.html index 46a2b4f1a..141234aca 100644 --- a/src/webui/service/templates/slice/home.html +++ b/src/webui/service/templates/slice/home.html @@ -31,20 +31,19 @@
#UUIDName Type End points Status + {{ service.name }} + {{ ste.Name(service.service_type).replace('SERVICETYPE_', '') }}
- + + - {% if slices %} {% for slice in slices %} - + +
#UUIDName End points Status
- {{ slice.slice_id.slice_uuid.uuid }} - {{ slice.slice_id.slice_uuid.uuid }}{{ slice.name }}
    {% for i in range(slice.slice_endpoint_ids|length) %} -- GitLab From c241ced146db9b3b716bf5d21b6c394329a8fdd3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Jan 2023 18:39:35 +0000 Subject: [PATCH 10/31] Monitoring component: - added checkpoints in CI/CD pipeline since it gets stuck for unknown reasons --- src/monitoring/.gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index 4a981cba2..a083134bc 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -60,13 +60,20 @@ unit_test monitoring: - sleep 30 - docker ps -a - docker logs $IMAGE_NAME + - echo "checkpoint-1" - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - echo "checkpoint-2" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + - echo "checkpoint-3" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: + - echo "checkpoint-4" - docker rm -f $IMAGE_NAME + - echo "checkpoint-5" - docker rm -f questdb + - echo "checkpoint-6" - docker network rm teraflowbridge + - echo "checkpoint-7" rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -- GitLab From 780e2197f0fc7596cab5914b1bba919b84e6c159 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 14:48:19 +0000 Subject: [PATCH 11/31] Monitoring component: - added log verbosity in CI/CD pipeline since it gets stuck for unknown reasons --- src/monitoring/.gitlab-ci.yml | 2 +- src/monitoring/tests/test_unitary.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index a083134bc..75cba475c 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -61,7 +61,7 @@ unit_test monitoring: - docker ps -a - docker logs $IMAGE_NAME - echo "checkpoint-1" - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO -o log_cli=true --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - echo "checkpoint-2" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - echo "checkpoint-3" diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index e70827cbc..b52e719ec 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -92,15 +92,25 @@ class MockContextService(GenericGrpcService): @pytest.fixture(scope='session') def context_service(): + LOGGER.info('Initializing MockContextService...') _service = MockContextService(MOCKSERVICE_PORT) _service.start() + + LOGGER.info('Yielding MockContextService...') yield _service + + LOGGER.info('Terminating MockContextService...') _service.stop() @pytest.fixture(scope='session') def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing ContextClient...') _client = ContextClient() + + LOGGER.info('Yielding ContextClient...') yield _client + + LOGGER.info('Closing ContextClient...') _client.close() @pytest.fixture(scope='session') @@ -120,8 +130,13 @@ def device_service(context_service : MockContextService): # pylint: disable=rede @pytest.fixture(scope='session') def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') yield _client + + LOGGER.info('Closing DeviceClient...') _client.close() # This fixture will be requested by test cases and last during testing session -- GitLab From 65f210b0d24e7c7f3f5ab0f28a39e932d2b3ed25 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 15:04:17 +0000 Subject: [PATCH 12/31] Monitoring component: - added log verbosity in CI/CD pipeline since it gets stuck for unknown reasons --- src/monitoring/service/EventTools.py | 10 +++++++++- src/monitoring/tests/test_unitary.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 189e78ce6..c881be163 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -15,7 +15,7 @@ import threading from queue import Queue -import grpc +import grpc, logging from common.method_wrappers.ServiceExceptions import ServiceException from context.client.ContextClient import ContextClient @@ -26,6 +26,8 @@ from monitoring.client.MonitoringClient import MonitoringClient from monitoring.service.MonitoringServiceServicerImpl import LOGGER from common.proto import monitoring_pb2 +LOGGER = logging.getLogger(__name__) + class EventsDeviceCollector: def __init__(self) -> None: # pylint: disable=redefined-outer-name self._events_queue = Queue() @@ -46,12 +48,15 @@ class EventsDeviceCollector: return False def _collect(self, events_stream): + LOGGER.warning('[_collect] begin') try: for event in events_stream: self._events_queue.put_nowait(event) except grpc.RpcError as e: if e.code() != grpc.StatusCode.CANCELLED: # pylint: disable=no-member + LOGGER.warning('[_collect] raise') raise # pragma: no cover + LOGGER.warning('[_collect] end') def start(self): try: @@ -63,8 +68,11 @@ class EventsDeviceCollector: return self._events_queue.get(block=block, timeout=timeout) def stop(self): + LOGGER.warning('[stop] begin') self._device_stream.cancel() + LOGGER.warning('[stop] joining') self._device_thread.join() + LOGGER.warning('[stop] end') def listen_events(self): try: diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index b52e719ec..4f0508ce5 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -102,6 +102,8 @@ def context_service(): LOGGER.info('Terminating MockContextService...') _service.stop() + LOGGER.info('Terminated MockContextService...') + @pytest.fixture(scope='session') def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument LOGGER.info('Initializing ContextClient...') @@ -113,6 +115,8 @@ def context_client(context_service : MockContextService): # pylint: disable=rede LOGGER.info('Closing ContextClient...') _client.close() + LOGGER.info('Closed ContextClient...') + @pytest.fixture(scope='session') def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument LOGGER.info('Initializing DeviceService...') @@ -128,6 +132,8 @@ def device_service(context_service : MockContextService): # pylint: disable=rede LOGGER.info('Terminating DeviceService...') _service.stop() + LOGGER.info('Terminated DeviceService...') + @pytest.fixture(scope='session') def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument LOGGER.info('Initializing DeviceClient...') @@ -139,6 +145,8 @@ def device_client(device_service : DeviceService): # pylint: disable=redefined-o LOGGER.info('Closing DeviceClient...') _client.close() + LOGGER.info('Closed DeviceClient...') + # This fixture will be requested by test cases and last during testing session @pytest.fixture(scope='session') def monitoring_service( @@ -156,6 +164,8 @@ def monitoring_service( LOGGER.info('Terminating MonitoringService...') _service.stop() + LOGGER.info('Terminated MonitoringService...') + # This fixture will be requested by test cases and last during testing session. # The client requires the server, so client fixture has the server as dependency. @pytest.fixture(scope='session') @@ -170,6 +180,8 @@ def monitoring_client(monitoring_service : MonitoringService): # pylint: disable LOGGER.info('Closing MonitoringClient...') _client.close() + LOGGER.info('Closed MonitoringClient...') + @pytest.fixture(scope='session') def management_db(): _management_db = ManagementDBTools.ManagementDB('monitoring.db') @@ -536,6 +548,8 @@ def test_events_tools( device_client.DeleteDevice(response) events_collector.stop() + LOGGER.warning('test_get_device_events end') + def test_get_device_events( context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument @@ -564,6 +578,8 @@ def test_get_device_events( device_client.DeleteDevice(response) events_collector.stop() + LOGGER.warning('test_get_device_events end') + def test_listen_events( context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument device_client : DeviceClient, # pylint: disable=redefined-outer-name @@ -589,3 +605,5 @@ def test_listen_events( device_client.DeleteDevice(response) events_collector.stop() + + LOGGER.warning('test_listen_events end') -- GitLab From 5a1efbe9312d3b176292588bfb158a7c5a4693f5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 15:20:54 +0000 Subject: [PATCH 13/31] Monitoring component: - added log verbosity in CI/CD pipeline since it gets stuck for unknown reasons --- src/monitoring/service/EventTools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index c881be163..9ebe0774c 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -75,12 +75,15 @@ class EventsDeviceCollector: LOGGER.warning('[stop] end') def listen_events(self): + LOGGER.warning('[listen_events] begin') try: kpi_id_list = [] while not self._events_queue.empty(): # LOGGER.info('getting Kpi by KpiID') + LOGGER.warning('[listen_events] waiting event') event = self.get_event(block=True) + LOGGER.warning('[listen_events] event received') if event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE: device = self._context_client.GetDevice(event.device_id) for j,end_point in enumerate(device.device_endpoints): @@ -98,6 +101,8 @@ class EventsDeviceCollector: kpi_id = self._monitoring_client.SetKpi(kpi_descriptor) kpi_id_list.append(kpi_id) + + LOGGER.warning('[listen_events] return') return kpi_id_list except ServiceException as e: LOGGER.exception('ListenEvents exception') -- GitLab From 13f6b4385da2103c70c7f77e74093d69042614b4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 16:19:37 +0000 Subject: [PATCH 14/31] Monitoring component: - added log verbosity in CI/CD pipeline since it gets stuck for unknown reasons --- src/monitoring/service/EventTools.py | 60 ++++++++++++++-------------- src/monitoring/service/__main__.py | 6 ++- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 9ebe0774c..95350ae70 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -12,25 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import threading -from queue import Queue - -import grpc, logging - +import grpc, logging, queue, threading from common.method_wrappers.ServiceExceptions import ServiceException -from context.client.ContextClient import ContextClient - +from common.proto import monitoring_pb2 from common.proto.context_pb2 import Empty, EventTypeEnum - +from context.client.ContextClient import ContextClient from monitoring.client.MonitoringClient import MonitoringClient from monitoring.service.MonitoringServiceServicerImpl import LOGGER -from common.proto import monitoring_pb2 LOGGER = logging.getLogger(__name__) class EventsDeviceCollector: def __init__(self) -> None: # pylint: disable=redefined-outer-name - self._events_queue = Queue() + self._events_queue = queue.Queue() self._context_client_grpc = ContextClient() self._device_stream = self._context_client_grpc.GetDeviceEvents(Empty()) @@ -79,28 +73,32 @@ class EventsDeviceCollector: try: kpi_id_list = [] - while not self._events_queue.empty(): + while True: # LOGGER.info('getting Kpi by KpiID') - LOGGER.warning('[listen_events] waiting event') - event = self.get_event(block=True) - LOGGER.warning('[listen_events] event received') - if event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE: - device = self._context_client.GetDevice(event.device_id) - for j,end_point in enumerate(device.device_endpoints): - #for i, value in enumerate(kpi_sample_types_pb2.KpiSampleType.values()): - for i, value in enumerate(end_point.kpi_sample_types): - #if value == kpi_sample_types_pb2.KpiSampleType.KPISAMPLETYPE_UNKNOWN: continue - - kpi_descriptor = monitoring_pb2.KpiDescriptor() - - kpi_descriptor.kpi_description = device.device_type - kpi_descriptor.kpi_sample_type = value - #kpi_descriptor.service_id.service_uuid.uuid = "" - kpi_descriptor.device_id.CopyFrom(device.device_id) - kpi_descriptor.endpoint_id.CopyFrom(end_point.endpoint_id) - - kpi_id = self._monitoring_client.SetKpi(kpi_descriptor) - kpi_id_list.append(kpi_id) + try: + LOGGER.warning('[listen_events] waiting event') + event = self.get_event(block=True, timeout=0.5) + LOGGER.warning('[listen_events] event received') + + if event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE: + device = self._context_client.GetDevice(event.device_id) + for j,end_point in enumerate(device.device_endpoints): + #for i, value in enumerate(kpi_sample_types_pb2.KpiSampleType.values()): + for i, value in enumerate(end_point.kpi_sample_types): + #if value == kpi_sample_types_pb2.KpiSampleType.KPISAMPLETYPE_UNKNOWN: continue + + kpi_descriptor = monitoring_pb2.KpiDescriptor() + + kpi_descriptor.kpi_description = device.device_type + kpi_descriptor.kpi_sample_type = value + #kpi_descriptor.service_id.service_uuid.uuid = "" + kpi_descriptor.device_id.CopyFrom(device.device_id) + kpi_descriptor.endpoint_id.CopyFrom(end_point.endpoint_id) + + kpi_id = self._monitoring_client.SetKpi(kpi_descriptor) + kpi_id_list.append(kpi_id) + except queue.Empty: + break LOGGER.warning('[listen_events] return') return kpi_id_list diff --git a/src/monitoring/service/__main__.py b/src/monitoring/service/__main__.py index 3334a860c..78764ea64 100644 --- a/src/monitoring/service/__main__.py +++ b/src/monitoring/service/__main__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, signal, sys, threading +import logging, signal, sys, threading, time from prometheus_client import start_http_server from common.Constants import ServiceNameEnum from common.Settings import ( @@ -35,6 +35,8 @@ def start_monitoring(): events_collector = EventsDeviceCollector() events_collector.start() + # TODO: redesign this method to be more clear and clean + # Iterate while terminate is not set while not terminate.is_set(): list_new_kpi_ids = events_collector.listen_events() @@ -48,6 +50,8 @@ def start_monitoring(): monitor_kpi_request.monitoring_window_s = 86400 monitor_kpi_request.sampling_rate_s = 30 events_collector._monitoring_client.MonitorKpi(monitor_kpi_request) + + time.sleep(0.5) # let other tasks run; do not overload CPU else: # Terminate is set, looping terminates LOGGER.warning("Stopping execution...") -- GitLab From 7860c44cd608fb0db4e9a4268db39d1ad36f6158 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 17:07:13 +0000 Subject: [PATCH 15/31] Monitoring component: - added explicit terminate in Context Mock - added missing KpiId in GetStreamKpi test --- src/monitoring/tests/Messages.py | 11 +++++++++++ src/monitoring/tests/test_unitary.py | 6 ++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/monitoring/tests/Messages.py b/src/monitoring/tests/Messages.py index f15cb5ec2..5530c720c 100644 --- a/src/monitoring/tests/Messages.py +++ b/src/monitoring/tests/Messages.py @@ -55,6 +55,17 @@ def create_kpi_request_c(): _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member return _create_kpi_request +def create_kpi_request_d(): + _create_kpi_request = monitoring_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member + return _create_kpi_request + def monitor_kpi_request(kpi_uuid, monitoring_window_s, sampling_rate_s): _monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() _monitor_kpi_request.kpi_id.kpi_id.uuid = kpi_uuid # pylint: disable=maybe-no-member diff --git a/src/monitoring/tests/test_unitary.py b/src/monitoring/tests/test_unitary.py index 4f0508ce5..d892b98b8 100644 --- a/src/monitoring/tests/test_unitary.py +++ b/src/monitoring/tests/test_unitary.py @@ -47,7 +47,7 @@ from monitoring.service.EventTools import EventsDeviceCollector from monitoring.service.MetricsDBTools import MetricsDB from monitoring.service.MonitoringService import MonitoringService #from monitoring.service.SubscriptionManager import SubscriptionManager -from monitoring.tests.Messages import create_kpi_request, include_kpi_request, monitor_kpi_request, \ +from monitoring.tests.Messages import create_kpi_request, create_kpi_request_d, include_kpi_request, monitor_kpi_request, \ create_kpi_request_c, kpi_query, subs_descriptor, alarm_descriptor, alarm_subscription #, create_kpi_request_b from monitoring.tests.Objects import DEVICE_DEV1, DEVICE_DEV1_CONNECT_RULES, DEVICE_DEV1_UUID @@ -100,6 +100,7 @@ def context_service(): yield _service LOGGER.info('Terminating MockContextService...') + _service.context_servicer.msg_broker.terminate() _service.stop() LOGGER.info('Terminated MockContextService...') @@ -400,7 +401,8 @@ def test_delete_alarm(monitoring_client): # pylint: disable=redefined-outer-name # Test case that makes use of client fixture to test server's GetStreamKpi method def test_get_stream_kpi(monitoring_client): # pylint: disable=redefined-outer-name LOGGER.warning('test_getstream_kpi begin') - response = monitoring_client.GetStreamKpi(monitoring_pb2.Kpi()) + _kpi_id = monitoring_client.SetKpi(create_kpi_request_d()) + response = monitoring_client.GetStreamKpi(_kpi_id) LOGGER.debug(str(response)) assert isinstance(response, _MultiThreadedRendezvous) -- GitLab From ebbbc1340b441097194fdee4e2b1e70b2215b040 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 17:26:41 +0000 Subject: [PATCH 16/31] Monitoring component: - removed checkpoints in CI/CD pipeline - removed log messages in classes --- src/monitoring/.gitlab-ci.yml | 9 +-------- src/monitoring/service/EventTools.py | 10 ---------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/src/monitoring/.gitlab-ci.yml b/src/monitoring/.gitlab-ci.yml index 75cba475c..4a981cba2 100644 --- a/src/monitoring/.gitlab-ci.yml +++ b/src/monitoring/.gitlab-ci.yml @@ -60,20 +60,13 @@ unit_test monitoring: - sleep 30 - docker ps -a - docker logs $IMAGE_NAME - - echo "checkpoint-1" - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO -o log_cli=true --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - - echo "checkpoint-2" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - - echo "checkpoint-3" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - - echo "checkpoint-4" - docker rm -f $IMAGE_NAME - - echo "checkpoint-5" - docker rm -f questdb - - echo "checkpoint-6" - docker network rm teraflowbridge - - echo "checkpoint-7" rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 95350ae70..221a0ddbf 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -42,15 +42,12 @@ class EventsDeviceCollector: return False def _collect(self, events_stream): - LOGGER.warning('[_collect] begin') try: for event in events_stream: self._events_queue.put_nowait(event) except grpc.RpcError as e: if e.code() != grpc.StatusCode.CANCELLED: # pylint: disable=no-member - LOGGER.warning('[_collect] raise') raise # pragma: no cover - LOGGER.warning('[_collect] end') def start(self): try: @@ -62,23 +59,17 @@ class EventsDeviceCollector: return self._events_queue.get(block=block, timeout=timeout) def stop(self): - LOGGER.warning('[stop] begin') self._device_stream.cancel() - LOGGER.warning('[stop] joining') self._device_thread.join() - LOGGER.warning('[stop] end') def listen_events(self): - LOGGER.warning('[listen_events] begin') try: kpi_id_list = [] while True: # LOGGER.info('getting Kpi by KpiID') try: - LOGGER.warning('[listen_events] waiting event') event = self.get_event(block=True, timeout=0.5) - LOGGER.warning('[listen_events] event received') if event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE: device = self._context_client.GetDevice(event.device_id) @@ -100,7 +91,6 @@ class EventsDeviceCollector: except queue.Empty: break - LOGGER.warning('[listen_events] return') return kpi_id_list except ServiceException as e: LOGGER.exception('ListenEvents exception') -- GitLab From 3b11d31e3d82fcfd51e2dc7858d3e7f57f29e42a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 18:33:39 +0000 Subject: [PATCH 17/31] Context component: - ConfigRules: improved management and added support for ACL rules - Constraints: improved management --- src/context/service/database/ConfigRule.py | 256 ++++++++------------- src/context/service/database/Constraint.py | 7 +- src/context/service/database/Device.py | 9 +- src/context/service/database/Service.py | 10 +- src/context/service/database/Slice.py | 12 +- 5 files changed, 114 insertions(+), 180 deletions(-) diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 5443e178c..e35f246b6 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime, logging +import datetime, json, logging from sqlalchemy import delete from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Set from common.proto.context_pb2 import ConfigRule from common.tools.grpc.Tools import grpc_message_to_json_string -from .models.enums.ConfigAction import grpc_to_enum__config_action +from .models.enums.ConfigAction import ORM_ConfigActionEnum, grpc_to_enum__config_action from .models.ConfigRuleModel import ConfigRuleKindEnum, ConfigRuleModel -from .uuids._Builder import get_uuid_random +from .uuids._Builder import get_uuid_from_string +from .uuids.EndPoint import endpoint_get_uuid LOGGER = logging.getLogger(__name__) @@ -31,173 +32,106 @@ def compose_config_rules_data( ) -> List[Dict]: dict_config_rules : List[Dict] = list() for position,config_rule in enumerate(config_rules): - configrule_uuid = get_uuid_random() str_kind = config_rule.WhichOneof('config_rule') + kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_config_rule = { - 'configrule_uuid': configrule_uuid, - 'position' : position, - 'kind' : ConfigRuleKindEnum._member_map_.get(str_kind.upper()), # pylint: disable=no-member - 'action' : grpc_to_enum__config_action(config_rule.action), - 'data' : grpc_message_to_json_string(getattr(config_rule, str_kind, {})), - 'created_at' : now, - 'updated_at' : now, + 'position' : position, + 'kind' : kind, + 'action' : grpc_to_enum__config_action(config_rule.action), + 'data' : grpc_message_to_json_string(getattr(config_rule, str_kind, {})), + 'created_at': now, + 'updated_at': now, } - if device_uuid is not None: dict_config_rule['device_uuid' ] = device_uuid - if service_uuid is not None: dict_config_rule['service_uuid'] = service_uuid - if slice_uuid is not None: dict_config_rule['slice_uuid' ] = slice_uuid + + parent_uuid = None + if device_uuid is not None: + dict_config_rule['device_uuid'] = device_uuid + parent_uuid = device_uuid + elif service_uuid is not None: + dict_config_rule['service_uuid'] = service_uuid + parent_uuid = service_uuid + elif slice_uuid is not None: + dict_config_rule['slice_uuid'] = slice_uuid + parent_uuid = slice_uuid + else: + MSG = 'Parent for ConfigRule({:s}) cannot be identified '+\ + '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' + str_config_rule = grpc_message_to_json_string(config_rule) + raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) + + configrule_name = None + if kind == ConfigRuleKindEnum.CUSTOM: + configrule_name = config_rule.custom.resource_key + elif kind == ConfigRuleKindEnum.ACL: + endpoint_uuid = endpoint_get_uuid(config_rule.acl.endpoint_id, allow_random=False) + rule_set_name = config_rule.acl.rule_set.name + configrule_name = '{:s}/{:s}'.format(endpoint_uuid, rule_set_name) + else: + MSG = 'Name for ConfigRule({:s}) cannot be inferred '+\ + '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' + str_config_rule = grpc_message_to_json_string(config_rule) + raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) + + configrule_uuid = get_uuid_from_string(configrule_name, prefix_for_name=parent_uuid) + dict_config_rule['configrule_uuid'] = configrule_uuid + dict_config_rules.append(dict_config_rule) return dict_config_rules def upsert_config_rules( session : Session, config_rules : List[Dict], device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None, -) -> List[bool]: - # TODO: do not delete all rules; just add-remove as needed - stmt = delete(ConfigRuleModel) - if device_uuid is not None: stmt = stmt.where(ConfigRuleModel.device_uuid == device_uuid ) - if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid) - if slice_uuid is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid == slice_uuid ) - session.execute(stmt) +) -> bool: + uuids_to_delete : Set[str] = set() + uuids_to_upsert : Dict[str, int] = dict() + rules_to_upsert : List[Dict] = list() + for config_rule in config_rules: + if config_rule['action'] == ORM_ConfigActionEnum.SET: + configrule_uuid = config_rule['configrule_uuid'] + position = uuids_to_upsert.get(configrule_uuid) + if position is None: + # if not added, add it + rules_to_upsert.append(config_rule) + uuids_to_upsert[config_rule['configrule_uuid']] = len(rules_to_upsert) - 1 + else: + # if already added, update occurrence + rules_to_upsert[position] = config_rule + elif config_rule['action'] == ORM_ConfigActionEnum.DELETE: + uuids_to_delete.add(config_rule['configrule_uuid']) + else: + MSG = 'Action for ConfigRule({:s}) is not supported '+\ + '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' + str_config_rule = json.dumps(config_rule) + raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) + + #LOGGER.warning('uuids_to_delete={:s}'.format(str(uuids_to_delete))) + #LOGGER.warning('rules_to_upsert={:s}'.format(str(rules_to_upsert))) - configrule_updates = [] - if len(config_rules) > 0: - stmt = insert(ConfigRuleModel).values(config_rules) - #stmt = stmt.on_conflict_do_update( - # index_elements=[ConfigRuleModel.configrule_uuid], - # set_=dict( - # updated_at = stmt.excluded.updated_at, - # ) - #) + delete_affected = False + upsert_affected = False + + if len(uuids_to_delete) > 0: + stmt = delete(ConfigRuleModel) + if device_uuid is not None: stmt = stmt.where(ConfigRuleModel.device_uuid == device_uuid ) + if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid) + if slice_uuid is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid == slice_uuid ) + stmt = stmt.where(ConfigRuleModel.configrule_uuid.in_(uuids_to_delete)) + configrule_deletes = session.execute(stmt)#.fetchall() + delete_affected = int(configrule_deletes.rowcount) > 0 + + if len(rules_to_upsert) > 0: + stmt = insert(ConfigRuleModel).values(rules_to_upsert) + stmt = stmt.on_conflict_do_update( + index_elements=[ConfigRuleModel.configrule_uuid], + set_=dict( + position = stmt.excluded.position, + action = stmt.excluded.action, + data = stmt.excluded.data, + updated_at = stmt.excluded.updated_at, + ) + ) stmt = stmt.returning(ConfigRuleModel.created_at, ConfigRuleModel.updated_at) configrule_updates = session.execute(stmt).fetchall() + upsert_affected = any([(updated_at > created_at) for created_at,updated_at in configrule_updates]) - return configrule_updates - -#Union_SpecificConfigRule = Union[ -# ConfigRuleCustomModel, ConfigRuleAclModel -#] -# -#def set_config_rule( -# database : Database, db_config : ConfigModel, position : int, resource_key : str, resource_value : str, -#): # -> Tuple[ConfigRuleModel, bool]: -# -# str_rule_key_hash = fast_hasher(resource_key) -# str_config_rule_key = key_to_str([db_config.config_uuid, str_rule_key_hash], separator=':') -# -# data = {'config_fk': db_config, 'position': position, 'action': ORM_ConfigActionEnum.SET, 'key': resource_key, -# 'value': resource_value} -# to_add = ConfigRuleModel(**data) -# -# result = database.create_or_update(to_add) -# return result -#Tuple_ConfigRuleSpecs = Tuple[Type, str, Dict, ConfigRuleKindEnum] -# -#def parse_config_rule_custom(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs: -# config_rule_class = ConfigRuleCustomModel -# str_config_rule_id = grpc_config_rule.custom.resource_key -# config_rule_data = { -# 'key' : grpc_config_rule.custom.resource_key, -# 'value': grpc_config_rule.custom.resource_value, -# } -# return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.CUSTOM -# -#def parse_config_rule_acl(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs: -# config_rule_class = ConfigRuleAclModel -# grpc_endpoint_id = grpc_config_rule.acl.endpoint_id -# grpc_rule_set = grpc_config_rule.acl.rule_set -# device_uuid = grpc_endpoint_id.device_id.device_uuid.uuid -# endpoint_uuid = grpc_endpoint_id.endpoint_uuid.uuid -# str_endpoint_key = '/'.join([device_uuid, endpoint_uuid]) -# #str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id) -# str_config_rule_id = ':'.join([str_endpoint_key, grpc_rule_set.name]) -# config_rule_data = { -# #'endpoint_fk': db_endpoint, -# 'endpoint_id': grpc_message_to_json_string(grpc_endpoint_id), -# 'acl_data': grpc_message_to_json_string(grpc_rule_set), -# } -# return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.ACL -# -#CONFIGRULE_PARSERS = { -# 'custom': parse_config_rule_custom, -# 'acl' : parse_config_rule_acl, -#} -# -#Union_ConfigRuleModel = Union[ -# ConfigRuleCustomModel, ConfigRuleAclModel, -#] -# -#def set_config_rule( -# database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule, position : int -#) -> Tuple[Union_ConfigRuleModel, bool]: -# grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule')) -# parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind) -# if parser is None: -# raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format( -# grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule))) -# -# # create specific ConfigRule -# config_rule_class, str_config_rule_id, config_rule_data, config_rule_kind = parser(database, grpc_config_rule) -# str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id])) -# str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':') -# result : Tuple[Union_ConfigRuleModel, bool] = update_or_create_object( -# database, config_rule_class, str_config_rule_key, config_rule_data) -# db_specific_config_rule, updated = result -# -# # create generic ConfigRule -# config_rule_fk_field_name = 'config_rule_{:s}_fk'.format(config_rule_kind.value) -# config_rule_data = { -# 'config_fk': db_config, 'kind': config_rule_kind, 'position': position, -# 'action': ORM_ConfigActionEnum.SET, -# config_rule_fk_field_name: db_specific_config_rule -# } -# result : Tuple[ConfigRuleModel, bool] = update_or_create_object( -# database, ConfigRuleModel, str_config_rule_key, config_rule_data) -# db_config_rule, updated = result -# -# return db_config_rule, updated -# -#def delete_config_rule( -# database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule -#) -> None: -# grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule')) -# parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind) -# if parser is None: -# raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format( -# grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule))) -# -# # delete generic config rules; self deletes specific config rule -# _, str_config_rule_id, _, config_rule_kind = parser(database, grpc_config_rule) -# str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id])) -# str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':') -# db_config_rule : Optional[ConfigRuleModel] = get_object( -# database, ConfigRuleModel, str_config_rule_key, raise_if_not_found=False) -# if db_config_rule is None: return -# db_config_rule.delete() -# -#def update_config( -# database : Database, db_parent_pk : str, config_name : str, grpc_config_rules -#) -> List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]]: -# -# str_config_key = key_to_str([config_name, db_parent_pk], separator=':') -# result : Tuple[ConfigModel, bool] = get_or_create_object(database, ConfigModel, str_config_key) -# db_config, created = result -# -# db_objects = [(db_config, created)] -# -# for position,grpc_config_rule in enumerate(grpc_config_rules): -# action = grpc_to_enum__config_action(grpc_config_rule.action) -# -# if action == ORM_ConfigActionEnum.SET: -# result : Tuple[ConfigRuleModel, bool] = set_config_rule( -# database, db_config, grpc_config_rule, position) -# db_config_rule, updated = result -# db_objects.append((db_config_rule, updated)) -# elif action == ORM_ConfigActionEnum.DELETE: -# delete_config_rule(database, db_config, grpc_config_rule) -# else: -# msg = 'Unsupported Action({:s}) for ConfigRule({:s})' -# str_action = str(ConfigActionEnum.Name(action)) -# str_config_rule = grpc_message_to_json_string(grpc_config_rule) -# raise AttributeError(msg.format(str_action, str_config_rule)) -# -# return db_objects + return delete_affected or upsert_affected diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index 2880c05a8..82629b25c 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -47,14 +47,14 @@ def compose_constraints_data( def upsert_constraints( session : Session, constraints : List[Dict], service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None -) -> List[bool]: +) -> bool: # TODO: do not delete all constraints; just add-remove as needed stmt = delete(ConstraintModel) if service_uuid is not None: stmt = stmt.where(ConstraintModel.service_uuid == service_uuid) if slice_uuid is not None: stmt = stmt.where(ConstraintModel.slice_uuid == slice_uuid ) session.execute(stmt) - constraint_updates = [] + changed = False if len(constraints) > 0: stmt = insert(ConstraintModel).values(constraints) #stmt = stmt.on_conflict_do_update( @@ -65,8 +65,9 @@ def upsert_constraints( #) stmt = stmt.returning(ConstraintModel.created_at, ConstraintModel.updated_at) constraint_updates = session.execute(stmt).fetchall() + changed = any([(updated_at > created_at) for created_at,updated_at in constraint_updates]) - return constraint_updates + return changed # def set_constraint(self, db_constraints: ConstraintsModel, grpc_constraint: Constraint, position: int diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 07d1c7606..cde8751b4 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -20,6 +20,7 @@ from sqlalchemy_cockroachdb import run_transaction from typing import Dict, List, Optional, Set, Tuple from common.method_wrappers.ServiceExceptions import InvalidArgumentException, NotFoundException from common.proto.context_pb2 import Device, DeviceId +from common.tools.grpc.Tools import grpc_message_to_json_string from common.tools.object_factory.Device import json_device_id from .models.DeviceModel import DeviceModel from .models.EndPointModel import EndPointModel @@ -136,6 +137,7 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: created_at,updated_at = session.execute(stmt).fetchone() updated = updated_at > created_at + updated_endpoints = False if len(endpoints_data) > 0: stmt = insert(EndPointModel).values(endpoints_data) stmt = stmt.on_conflict_do_update( @@ -149,17 +151,16 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: ) stmt = stmt.returning(EndPointModel.created_at, EndPointModel.updated_at) endpoint_updates = session.execute(stmt).fetchall() - updated = updated or any([(updated_at > created_at) for created_at,updated_at in endpoint_updates]) + updated_endpoints = any([(updated_at > created_at) for created_at,updated_at in endpoint_updates]) if len(related_topologies) > 0: session.execute(insert(TopologyDeviceModel).values(related_topologies).on_conflict_do_nothing( index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid] )) - configrule_updates = upsert_config_rules(session, config_rules, device_uuid=device_uuid) - updated = updated or any([(updated_at > created_at) for created_at,updated_at in configrule_updates]) + changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid) - return updated + return updated or updated_endpoints or changed_config_rules updated = run_transaction(sessionmaker(bind=db_engine), callback) return json_device_id(device_uuid),updated diff --git a/src/context/service/database/Service.py b/src/context/service/database/Service.py index 76a830535..9b9e9a621 100644 --- a/src/context/service/database/Service.py +++ b/src/context/service/database/Service.py @@ -118,6 +118,7 @@ def service_set(db_engine : Engine, request : Service) -> Tuple[Dict, bool]: created_at,updated_at = session.execute(stmt).fetchone() updated = updated_at > created_at + # TODO: check if endpoints are changed if len(service_endpoints_data) > 0: stmt = insert(ServiceEndPointModel).values(service_endpoints_data) stmt = stmt.on_conflict_do_nothing( @@ -125,13 +126,10 @@ def service_set(db_engine : Engine, request : Service) -> Tuple[Dict, bool]: ) session.execute(stmt) - constraint_updates = upsert_constraints(session, constraints, service_uuid=service_uuid) - updated = updated or any([(updated_at > created_at) for created_at,updated_at in constraint_updates]) + changed_constraints = upsert_constraints(session, constraints, service_uuid=service_uuid) + changed_config_rules = upsert_config_rules(session, config_rules, service_uuid=service_uuid) - configrule_updates = upsert_config_rules(session, config_rules, service_uuid=service_uuid) - updated = updated or any([(updated_at > created_at) for created_at,updated_at in configrule_updates]) - - return updated + return updated or changed_constraints or changed_config_rules updated = run_transaction(sessionmaker(bind=db_engine), callback) return json_service_id(service_uuid, json_context_id(context_uuid)),updated diff --git a/src/context/service/database/Slice.py b/src/context/service/database/Slice.py index 84bfff343..113af9aa4 100644 --- a/src/context/service/database/Slice.py +++ b/src/context/service/database/Slice.py @@ -136,6 +136,7 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]: created_at,updated_at = session.execute(stmt).fetchone() updated = updated_at > created_at + # TODO: check if endpoints are changed if len(slice_endpoints_data) > 0: stmt = insert(SliceEndPointModel).values(slice_endpoints_data) stmt = stmt.on_conflict_do_nothing( @@ -143,6 +144,7 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]: ) session.execute(stmt) + # TODO: check if services are changed if len(slice_services_data) > 0: stmt = insert(SliceServiceModel).values(slice_services_data) stmt = stmt.on_conflict_do_nothing( @@ -150,6 +152,7 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]: ) session.execute(stmt) + # TODO: check if subslices are changed if len(slice_subslices_data) > 0: stmt = insert(SliceSubSliceModel).values(slice_subslices_data) stmt = stmt.on_conflict_do_nothing( @@ -157,13 +160,10 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]: ) session.execute(stmt) - constraint_updates = upsert_constraints(session, constraints, slice_uuid=slice_uuid) - updated = updated or any([(updated_at > created_at) for created_at,updated_at in constraint_updates]) + changed_constraints = upsert_constraints(session, constraints, slice_uuid=slice_uuid) + changed_config_rules = upsert_config_rules(session, config_rules, slice_uuid=slice_uuid) - configrule_updates = upsert_config_rules(session, config_rules, slice_uuid=slice_uuid) - updated = updated or any([(updated_at > created_at) for created_at,updated_at in configrule_updates]) - - return updated + return updated or changed_constraints or changed_config_rules updated = run_transaction(sessionmaker(bind=db_engine), callback) return json_slice_id(slice_uuid, json_context_id(context_uuid)),updated -- GitLab From 83d3a5f69d222c21c079bacf865297f56b207b83 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 18:35:33 +0000 Subject: [PATCH 18/31] Device component: - Improved update of ConfigRules changed in Context --- src/device/service/DeviceServiceServicerImpl.py | 6 +----- src/device/service/Tools.py | 10 ++++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index 179b7795b..5b537609b 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -121,11 +121,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): for error in errors: LOGGER.error(error) raise OperationFailedException('ConfigureDevice', extra_details=errors) - # Rules updated by configure_rules() and deconfigure_rules() methods. - # Code to be removed soon if not needed. - del device.device_config.config_rules[:] - populate_config_rules(device, driver) - + # Note: Rules are updated by configure_rules() and deconfigure_rules() methods. device_id = context_client.SetDevice(device) return device_id finally: diff --git a/src/device/service/Tools.py b/src/device/service/Tools.py index d2cd0b481..b4dc66dde 100644 --- a/src/device/service/Tools.py +++ b/src/device/service/Tools.py @@ -107,7 +107,7 @@ def populate_endpoints(device : Device, driver : _Driver, monitoring_loops : Mon return errors def _raw_config_rules_to_grpc( - device_uuid : str, device_config : DeviceConfig, error_template : str, default_config_action : ConfigActionEnum, + device_uuid : str, device_config : DeviceConfig, error_template : str, config_action : ConfigActionEnum, raw_config_rules : List[Tuple[str, Union[Any, Exception, None]]] ) -> List[str]: errors : List[str] = list() @@ -118,7 +118,7 @@ def _raw_config_rules_to_grpc( continue config_rule = device_config.config_rules.add() - config_rule.action = default_config_action + config_rule.action = config_action config_rule.custom.resource_key = resource_key config_rule.custom.resource_value = \ resource_value if isinstance(resource_value, str) else json.dumps(resource_value, sort_keys=True) @@ -177,9 +177,8 @@ def configure_rules(device : Device, driver : _Driver, resources_to_set : List[T for (resource_key, resource_value), result in zip(resources_to_set, results_setconfig) ] - device_config = DeviceConfig() # ignored; added at the end of ConfigureDevice return _raw_config_rules_to_grpc( - device_uuid, device_config, ERROR_SET, ConfigActionEnum.CONFIGACTION_SET, results_setconfig) + device_uuid, device.device_config, ERROR_SET, ConfigActionEnum.CONFIGACTION_SET, results_setconfig) def deconfigure_rules(device : Device, driver : _Driver, resources_to_delete : List[Tuple[str, Any]]) -> List[str]: device_uuid = device.device_id.device_uuid.uuid @@ -190,9 +189,8 @@ def deconfigure_rules(device : Device, driver : _Driver, resources_to_delete : L for (resource_key, resource_value), result in zip(resources_to_delete, results_deleteconfig) ] - device_config = DeviceConfig() # ignored; added at the end of ConfigureDevice return _raw_config_rules_to_grpc( - device_uuid, device_config, ERROR_DELETE, ConfigActionEnum.CONFIGACTION_DELETE, results_deleteconfig) + device_uuid, device.device_config, ERROR_DELETE, ConfigActionEnum.CONFIGACTION_DELETE, results_deleteconfig) def subscribe_kpi(request : MonitoringSettings, driver : _Driver, monitoring_loops : MonitoringLoops) -> List[str]: kpi_uuid = request.kpi_id.kpi_id.uuid -- GitLab From 2799209f23ff8675221e0a2ced35a37df0e588bd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 18:38:55 +0000 Subject: [PATCH 19/31] Service component: - Update ConfigRules in Context when UpdateService is executed - Defined SettingsHandler class to help in retrieving settings - Updated ServiceHandlers to use the new SettingsHandler class --- .../service/ServiceServiceServicerImpl.py | 16 ++-- .../service_handler_api/SettingsHandler.py | 88 +++++++++++++++++++ .../service/service_handler_api/Tools.py | 11 +++ .../L2NMEmulatedServiceHandler.py | 61 +++++-------- .../L2NMOpenConfigServiceHandler.py | 61 +++++-------- .../L3NMEmulatedServiceHandler.py | 72 +++++++-------- .../L3NMOpenConfigServiceHandler.py | 61 +++++-------- .../microwave/MicrowaveServiceHandler.py | 68 +++++--------- .../tapi_tapi/TapiServiceHandler.py | 61 +++++-------- 9 files changed, 259 insertions(+), 240 deletions(-) create mode 100644 src/service/service/service_handler_api/SettingsHandler.py diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 50f645e42..87eda2678 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -95,13 +95,17 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): service.CopyFrom(request if _service is None else _service) service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED # pylint: disable=no-member - del service.service_endpoint_ids[:] # pylint: disable=no-member - for service_endpoint_id in request.service_endpoint_ids: - service.service_endpoint_ids.add().CopyFrom(service_endpoint_id) # pylint: disable=no-member + del service.service_endpoint_ids[:] # pylint: disable=no-member + for endpoint_id in request.service_endpoint_ids: + service.service_endpoint_ids.add().CopyFrom(endpoint_id) # pylint: disable=no-member - del service.service_constraints[:] # pylint: disable=no-member - for service_constraint in request.service_constraints: - service.service_constraints.add().CopyFrom(service_constraint) # pylint: disable=no-member + del service.service_constraints[:] # pylint: disable=no-member + for constraint in request.service_constraints: + service.service_constraints.add().CopyFrom(constraint) # pylint: disable=no-member + + del service.service_config.config_rules[:] # pylint: disable=no-member + for config_rule in request.service_config.config_rules: + service.service_config.config_rules.add().CopyFrom(config_rule) # pylint: disable=no-member service_id_with_uuids = context_client.SetService(service) service_with_uuids = context_client.GetService(service_id_with_uuids) diff --git a/src/service/service/service_handler_api/SettingsHandler.py b/src/service/service/service_handler_api/SettingsHandler.py new file mode 100644 index 000000000..4df24cee0 --- /dev/null +++ b/src/service/service/service_handler_api/SettingsHandler.py @@ -0,0 +1,88 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import anytree, json, logging +from typing import Any, List, Optional, Tuple, Union +from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, Device, EndPoint, ServiceConfig +from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string +from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value + +LOGGER = logging.getLogger(__name__) + +class SettingsHandler: + def __init__(self, service_config : ServiceConfig, **settings) -> None: + self.__resolver = anytree.Resolver(pathattr='name') + self.__config = TreeNode('.') + for config_rule in service_config.config_rules: + self.update_config_rule(config_rule) + + @staticmethod + def _config_rule_to_raw(config_rule : ConfigRule) -> Optional[Tuple[int, str, Any]]: + action = config_rule.action + kind = config_rule.WhichOneof('config_rule') + if kind == 'custom': + key_or_path = config_rule.custom.resource_key + value = config_rule.custom.resource_value + try: + value = json.loads(value) + except: # pylint: disable=bare-except + pass + elif kind == 'acl': + device_uuid = config_rule.acl.endpoint_id.device_id.device_uuid.uuid + endpoint_uuid = config_rule.acl.endpoint_id.endpoint_uuid.uuid + acl_ruleset_name = config_rule.acl.rule_set.name + ACL_KEY_TEMPLATE = '/device[{:s}]/endpoint[{:s}]/acl_ruleset[{:s}]' + key_or_path = ACL_KEY_TEMPLATE.format(device_uuid, endpoint_uuid, acl_ruleset_name) + value = grpc_message_to_json(config_rule.acl) + else: + MSG = 'Unsupported Kind({:s}) in ConfigRule({:s})' + LOGGER.warning(MSG.format(str(kind), grpc_message_to_json_string(config_rule))) + return None + + return action, key_or_path, value + + def get(self, key_or_path : Union[str, List[str]], default : Optional[Any] = None) -> Optional[TreeNode]: + return get_subnode(self.__resolver, self.__config, key_or_path, default=default) + + def get_endpoint_settings(self, device : Device, endpoint : EndPoint) -> Optional[TreeNode]: + device_keys = device.device_id.device_uuid.uuid, device.name + endpoint_keys = endpoint.endpoint_id.endpoint_uuid.uuid, endpoint.name + + for device_key in device_keys: + for endpoint_key in endpoint_keys: + endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_key, endpoint_key) + endpoint_settings = self.get(endpoint_settings_uri) + if endpoint_settings is not None: return endpoint_settings + + return None + + def set(self, key_or_path : Union[str, List[str]], value : Any) -> None: + set_subnode_value(self.__resolver, self.__config, key_or_path, value) + + def delete(self, key_or_path : Union[str, List[str]]) -> None: + delete_subnode(self.__resolver, self.__config, key_or_path) + + def update_config_rule(self, config_rule : ConfigRule) -> None: + raw_data = SettingsHandler._config_rule_to_raw(config_rule) + if raw_data is None: return + action, key_or_path, value = raw_data + + if action == ConfigActionEnum.CONFIGACTION_SET: + self.set(key_or_path, value) + elif action == ConfigActionEnum.CONFIGACTION_DELETE: + self.delete(key_or_path) + else: + MSG = 'Unsupported Action({:s}) in ConfigRule({:s})' + LOGGER.warning(MSG.format(str(action), grpc_message_to_json_string(config_rule))) + return diff --git a/src/service/service/service_handler_api/Tools.py b/src/service/service/service_handler_api/Tools.py index 61ad79761..ebd16a532 100644 --- a/src/service/service/service_handler_api/Tools.py +++ b/src/service/service/service_handler_api/Tools.py @@ -14,6 +14,8 @@ import functools from typing import Any, List, Union +from common.method_wrappers.ServiceExceptions import NotFoundException +from common.proto.context_pb2 import Device, EndPoint ACTION_MSG_SET_ENDPOINT = 'Set EndPoint(device_uuid={:s}, endpoint_uuid={:s}, topology_uuid={:s})' ACTION_MSG_DELETE_ENDPOINT = 'Delete EndPoint(device_uuid={:s}, endpoint_uuid={:s}, topology_uuid={:s})' @@ -40,3 +42,12 @@ check_errors_setconstraint = functools.partial(_check_errors, ACTION_MSG_SET_ check_errors_deleteconstraint = functools.partial(_check_errors, ACTION_MSG_DELETE_CONSTRAINT) check_errors_setconfig = functools.partial(_check_errors, ACTION_MSG_SET_CONFIG ) check_errors_deleteconfig = functools.partial(_check_errors, ACTION_MSG_DELETE_CONFIG ) + +def get_endpoint_matching(device : Device, endpoint_uuid_or_name : str) -> EndPoint: + for endpoint in device.device_endpoints: + choices = {endpoint.endpoint_id.endpoint_uuid.uuid, endpoint.name} + if endpoint_uuid_or_name in choices: return endpoint + + device_uuid = device.device_id.device_uuid.uuid + extra_details = 'Device({:s})'.format(str(device_uuid)) + raise NotFoundException('Endpoint', endpoint_uuid_or_name, extra_details=extra_details) diff --git a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py index bc628c160..66259d1f6 100644 --- a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type +from service.service.service_handler_api.Tools import get_endpoint_matching from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor from .ConfigRules import setup_config_rules, teardown_config_rules @@ -47,22 +48,8 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( @@ -72,7 +59,7 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -81,17 +68,17 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = setup_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetEndpoint({:s})'.format(str(endpoint))) @@ -107,7 +94,7 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -116,17 +103,17 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = teardown_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint({:s})'.format(str(endpoint))) @@ -160,9 +147,8 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -178,8 +164,7 @@ class L2NMEmulatedServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) diff --git a/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py index 23df44413..63442a6b4 100644 --- a/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type +from service.service.service_handler_api.Tools import get_endpoint_matching from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor from .ConfigRules import setup_config_rules, teardown_config_rules @@ -47,22 +48,8 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( @@ -72,7 +59,7 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -81,17 +68,17 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = setup_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetEndpoint({:s})'.format(str(endpoint))) @@ -107,7 +94,7 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -116,17 +103,17 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = teardown_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint({:s})'.format(str(endpoint))) @@ -160,9 +147,8 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -178,8 +164,7 @@ class L2NMOpenConfigServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) diff --git a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py index f16122519..8a39ed474 100644 --- a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type +from service.service.service_handler_api.Tools import get_endpoint_matching from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor from .ConfigRules import setup_config_rules, teardown_config_rules @@ -47,67 +48,64 @@ class L3NMEmulatedServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( self, endpoints : List[Tuple[str, str, Optional[str]]], connection_uuid : Optional[str] = None ) -> List[Union[bool, Exception]]: + LOGGER.info('[SetEndpoint] endpoints={:s}'.format(str(endpoints))) + LOGGER.info('[SetEndpoint] connection_uuid={:s}'.format(str(connection_uuid))) + chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') + LOGGER.info('[SetEndpoint] settings={:s}'.format(str(settings))) results = [] for endpoint in endpoints: + LOGGER.info('[SetEndpoint] endpoint={:s}'.format(str(endpoint))) try: chk_type('endpoint', endpoint, (tuple, list)) chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) + LOGGER.info('[SetEndpoint] endpoint_settings={:s}'.format(str(endpoint_settings))) json_config_rules = setup_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) + LOGGER.info('[SetEndpoint] json_config_rules={:s}'.format(str(json_config_rules))) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetEndpoint({:s})'.format(str(endpoint))) results.append(e) + LOGGER.info('[SetEndpoint] results={:s}'.format(str(results))) return results @metered_subclass_method(METRICS_POOL) def DeleteEndpoint( self, endpoints : List[Tuple[str, str, Optional[str]]], connection_uuid : Optional[str] = None ) -> List[Union[bool, Exception]]: + LOGGER.info('[DeleteEndpoint] endpoints={:s}'.format(str(endpoints))) + LOGGER.info('[DeleteEndpoint] connection_uuid={:s}'.format(str(connection_uuid))) + chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -116,17 +114,17 @@ class L3NMEmulatedServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = teardown_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint({:s})'.format(str(endpoint))) @@ -160,9 +158,8 @@ class L3NMEmulatedServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -178,8 +175,7 @@ class L3NMEmulatedServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) diff --git a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py index 0f5cb6c55..3dc98f71b 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type +from service.service.service_handler_api.Tools import get_endpoint_matching from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor from .ConfigRules import setup_config_rules, teardown_config_rules @@ -47,22 +48,8 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( @@ -72,7 +59,7 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -81,17 +68,17 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = setup_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetEndpoint({:s})'.format(str(endpoint))) @@ -107,7 +94,7 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): if len(endpoints) == 0: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) + settings = self.__settings_handler.get('/settings') results = [] for endpoint in endpoints: @@ -116,17 +103,17 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): chk_length('endpoint', endpoint, min_length=2, max_length=3) device_uuid, endpoint_uuid = endpoint[0:2] # ignore topology_uuid by now - endpoint_settings_uri = '/device[{:s}]/endpoint[{:s}]/settings'.format(device_uuid, endpoint_uuid) - endpoint_settings : TreeNode = get_subnode(self.__resolver, self.__config, endpoint_settings_uri, None) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) json_config_rules = teardown_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, settings, endpoint_settings) - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - del device.device_config.config_rules[:] + del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint({:s})'.format(str(endpoint))) @@ -160,9 +147,8 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -178,8 +164,7 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) diff --git a/src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py b/src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py index fb54a1bc1..a16f8cdfa 100644 --- a/src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py +++ b/src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, Dict, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor LOGGER = logging.getLogger(__name__) @@ -38,22 +38,8 @@ class MicrowaveServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( @@ -61,26 +47,22 @@ class MicrowaveServiceHandler(_ServiceHandler): ) -> List[Union[bool, Exception]]: LOGGER.info('[SetEndpoint] endpoints={:s}'.format(str(endpoints))) LOGGER.info('[SetEndpoint] connection_uuid={:s}'.format(str(connection_uuid))) + chk_type('endpoints', endpoints, list) + if len(endpoints) != 2: return [] service_uuid = self.__service.service_id.service_uuid.uuid + settings = self.__settings_handler.get('/settings') + json_settings : Dict = {} if settings is None else settings.value + vlan_id = json_settings.get('vlan_id', 121) results = [] try: - chk_type('endpoints', endpoints, list) - if len(endpoints) != 2: raise Exception('len(endpoints) != 2') - - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) - if settings is None: - raise Exception('Unable to retrieve settings for Service({:s})'.format(str(service_uuid))) - - json_settings : Dict = settings.value - vlan_id = json_settings.get('vlan_id', 121) # endpoints are retrieved in the following format --> '/endpoints/endpoint[172.26.60.243:9]' node_id_src, tp_id_src = check_endpoint(endpoints[0][1], service_uuid) node_id_dst, tp_id_dst = check_endpoint(endpoints[1][1], service_uuid) device_uuid = endpoints[0][0] - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) json_config_rule = json_config_rule_set('/services/service[{:s}]'.format(service_uuid), { 'uuid' : service_uuid, 'node_id_src': node_id_src, @@ -89,9 +71,9 @@ class MicrowaveServiceHandler(_ServiceHandler): 'tp_id_dst' : tp_id_dst, 'vlan_id' : vlan_id, }) - del device.device_config.config_rules[:] - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + del device_obj.device_config.config_rules[:] + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetEndpoint for Service({:s})'.format(str(service_uuid))) @@ -106,21 +88,21 @@ class MicrowaveServiceHandler(_ServiceHandler): LOGGER.info('[DeleteEndpoint] endpoints={:s}'.format(str(endpoints))) LOGGER.info('[DeleteEndpoint] connection_uuid={:s}'.format(str(connection_uuid))) + chk_type('endpoints', endpoints, list) + if len(endpoints) != 2: return [] + service_uuid = self.__service.service_id.service_uuid.uuid results = [] try: - chk_type('endpoints', endpoints, list) - if len(endpoints) < 1: raise Exception('len(endpoints) < 1') - device_uuid = endpoints[0][0] - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) json_config_rule = json_config_rule_delete('/services/service[{:s}]'.format(service_uuid), { 'uuid': service_uuid }) - del device.device_config.config_rules[:] - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + del device_obj.device_config.config_rules[:] + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint for Service({:s})'.format(str(service_uuid))) @@ -154,9 +136,8 @@ class MicrowaveServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -172,8 +153,7 @@ class MicrowaveServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) diff --git a/src/service/service/service_handlers/tapi_tapi/TapiServiceHandler.py b/src/service/service/service_handlers/tapi_tapi/TapiServiceHandler.py index 24371203a..d8a4668bb 100644 --- a/src/service/service/service_handlers/tapi_tapi/TapiServiceHandler.py +++ b/src/service/service/service_handlers/tapi_tapi/TapiServiceHandler.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import anytree, json, logging +import json, logging from typing import Any, Dict, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type from service.service.service_handler_api._ServiceHandler import _ServiceHandler -from service.service.service_handler_api.AnyTreeTools import TreeNode, delete_subnode, get_subnode, set_subnode_value +from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.task_scheduler.TaskExecutor import TaskExecutor LOGGER = logging.getLogger(__name__) @@ -32,22 +32,8 @@ class TapiServiceHandler(_ServiceHandler): self, service : Service, task_executor : TaskExecutor, **settings ) -> None: self.__service = service - self.__task_executor = task_executor # pylint: disable=unused-private-member - self.__resolver = anytree.Resolver(pathattr='name') - self.__config = TreeNode('.') - for config_rule in service.service_config.config_rules: - action = config_rule.action - if config_rule.WhichOneof('config_rule') != 'custom': continue - resource_key = config_rule.custom.resource_key - resource_value = config_rule.custom.resource_value - if action == ConfigActionEnum.CONFIGACTION_SET: - try: - resource_value = json.loads(resource_value) - except: # pylint: disable=bare-except - pass - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) - elif action == ConfigActionEnum.CONFIGACTION_DELETE: - delete_subnode(self.__resolver, self.__config, resource_key) + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) @metered_subclass_method(METRICS_POOL) def SetEndpoint( @@ -59,10 +45,8 @@ class TapiServiceHandler(_ServiceHandler): if len(endpoints) != 2: return [] service_uuid = self.__service.service_id.service_uuid.uuid - settings : TreeNode = get_subnode(self.__resolver, self.__config, '/settings', None) - if settings is None: raise Exception('Unable to retrieve settings for Service({:s})'.format(str(service_uuid))) - - json_settings : Dict = settings.value + settings = self.__settings_handler.get('/settings') + json_settings : Dict = {} if settings is None else settings.value capacity_value = json_settings.get('capacity_value', 50.0) capacity_unit = json_settings.get('capacity_unit', 'GHz') layer_proto_name = json_settings.get('layer_proto_name', 'PHOTONIC_MEDIA') @@ -72,7 +56,7 @@ class TapiServiceHandler(_ServiceHandler): results = [] try: device_uuid = endpoints[0][0] - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) json_config_rule = json_config_rule_set('/service[{:s}]'.format(service_uuid), { 'uuid' : service_uuid, 'input_sip' : endpoints[0][1], @@ -83,12 +67,12 @@ class TapiServiceHandler(_ServiceHandler): 'layer_protocol_qualifier': layer_proto_qual, 'direction' : direction, }) - del device.device_config.config_rules[:] - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + del device_obj.device_config.config_rules[:] + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except - LOGGER.exception('Unable to configure Service({:s})'.format(str(service_uuid))) + LOGGER.exception('Unable to SetEndpoint for Service({:s})'.format(str(service_uuid))) results.append(e) return results @@ -104,14 +88,17 @@ class TapiServiceHandler(_ServiceHandler): if len(endpoints) != 2: return [] service_uuid = self.__service.service_id.service_uuid.uuid + results = [] try: device_uuid = endpoints[0][0] - device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) - json_config_rule = json_config_rule_delete('/service[{:s}]'.format(service_uuid), {'uuid': service_uuid}) - del device.device_config.config_rules[:] - device.device_config.config_rules.append(ConfigRule(**json_config_rule)) - self.__task_executor.configure_device(device) + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + json_config_rule = json_config_rule_delete('/service[{:s}]'.format(service_uuid), { + 'uuid': service_uuid + }) + del device_obj.device_config.config_rules[:] + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteEndpoint for Service({:s})'.format(str(service_uuid))) @@ -145,9 +132,8 @@ class TapiServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, resource_value = resource - resource_value = json.loads(resource_value) - set_subnode_value(self.__resolver, self.__config, resource_key, resource_value) + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) @@ -163,8 +149,7 @@ class TapiServiceHandler(_ServiceHandler): results = [] for resource in resources: try: - resource_key, _ = resource - delete_subnode(self.__resolver, self.__config, resource_key) + self.__settings_handler.delete(resource[0]) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) results.append(e) -- GitLab From 39436a6b026221b8e774dfb06fd0958a5d67c027 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 28 Jan 2023 23:49:55 +0000 Subject: [PATCH 20/31] Device component: - testing CI pipeline, added log messages --- src/device/tests/test_unitary_emulated.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/device/tests/test_unitary_emulated.py b/src/device/tests/test_unitary_emulated.py index 8a1b30a6e..1af1ad861 100644 --- a/src/device/tests/test_unitary_emulated.py +++ b/src/device/tests/test_unitary_emulated.py @@ -156,8 +156,8 @@ def test_device_emulated_configure( for config_rule in device_data.device_config.config_rules if config_rule.WhichOneof('config_rule') == 'custom' ] - #LOGGER.info('device_data.device_config.config_rules = \n{:s}'.format( - # '\n'.join(['{:s} {:s} = {:s}'.format(*config_rule) for config_rule in config_rules]))) + LOGGER.info('device_data.device_config.config_rules = \n{:s}'.format( + '\n'.join(['{:s} {:s} = {:s}'.format(*config_rule) for config_rule in config_rules]))) RESULTING_CONFIG_ENDPOINTS = {cr['custom']['resource_key']:cr for cr in copy.deepcopy(DEVICE_EMU_CONFIG_ENDPOINTS)} for endpoint_cooked in DEVICE_EMU_ENDPOINTS_COOKED: values = json.loads(RESULTING_CONFIG_ENDPOINTS[endpoint_cooked[0]]['custom']['resource_value']) @@ -168,14 +168,14 @@ def test_device_emulated_configure( config_rule = ( ConfigActionEnum.Name(config_rule['action']), config_rule['custom']['resource_key'], json.loads(json.dumps(config_rule['custom']['resource_value']))) - #LOGGER.info('config_rule: {:s} {:s} = {:s}'.format(*config_rule)) + LOGGER.info('A config_rule: {:s} {:s} = {:s}'.format(*config_rule)) assert config_rule in config_rules for config_rule in DEVICE_EMU_CONFIG_ADDRESSES: assert 'custom' in config_rule config_rule = ( ConfigActionEnum.Name(config_rule['action']), config_rule['custom']['resource_key'], json.loads(json.dumps(config_rule['custom']['resource_value']))) - #LOGGER.info('config_rule: {:s} {:s} = {:s}'.format(*config_rule)) + LOGGER.info('B config_rule: {:s} {:s} = {:s}'.format(*config_rule)) assert config_rule in config_rules # Try to reconfigure... -- GitLab From ba9ab833b0f3a7bbf5989992b95c984f9aff63d5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 08:36:44 +0000 Subject: [PATCH 21/31] Common: - updated method to update config rules and support action updates --- src/common/tools/grpc/ConfigRules.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/common/tools/grpc/ConfigRules.py b/src/common/tools/grpc/ConfigRules.py index e109cb7a0..4a7e588e4 100644 --- a/src/common/tools/grpc/ConfigRules.py +++ b/src/common/tools/grpc/ConfigRules.py @@ -18,24 +18,31 @@ import json from typing import Any, Dict, Tuple -from common.proto.context_pb2 import ConfigActionEnum, ConfigRule +from common.proto.context_pb2 import ConfigActionEnum from common.tools.grpc.Tools import grpc_message_to_json_string -def update_config_rule_custom(config_rules, resource_key : str, fields : Dict[str, Tuple[Any, bool]]) -> ConfigRule: +def update_config_rule_custom( + config_rules, resource_key : str, fields : Dict[str, Tuple[Any, bool]], + new_action : ConfigActionEnum = ConfigActionEnum.CONFIGACTION_SET +) -> None: # fields: Dict[field_name : str, Tuple[field_value : Any, raise_if_differs : bool]] + # TODO: add support for ACL config rules + for config_rule in config_rules: - if config_rule.WhichOneof('config_rule') != 'custom': continue + kind = config_rule.WhichOneof('config_rule') + if kind != 'custom': continue if config_rule.custom.resource_key != resource_key: continue json_resource_value = json.loads(config_rule.custom.resource_value) break # found, end loop else: # not found, add it config_rule = config_rules.add() # pylint: disable=no-member - config_rule.action = ConfigActionEnum.CONFIGACTION_SET config_rule.custom.resource_key = resource_key json_resource_value = {} + config_rule.action = new_action + for field_name,(field_value, raise_if_differs) in fields.items(): if (field_name not in json_resource_value) or not raise_if_differs: # missing or raise_if_differs=False, add/update it -- GitLab From 1dbb33d24ec195086a3e4329e9f988c85ef314ff Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 08:37:27 +0000 Subject: [PATCH 22/31] Device component: - implemented support to compose resulting rules considering previous values --- src/device/service/Tools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/device/service/Tools.py b/src/device/service/Tools.py index b4dc66dde..28b5391a7 100644 --- a/src/device/service/Tools.py +++ b/src/device/service/Tools.py @@ -19,6 +19,7 @@ from common.method_wrappers.ServiceExceptions import InvalidArgumentException from common.proto.context_pb2 import ConfigActionEnum, Device, DeviceConfig from common.proto.device_pb2 import MonitoringSettings from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.grpc.ConfigRules import update_config_rule_custom from common.tools.grpc.Tools import grpc_message_to_json from .driver_api._Driver import _Driver, RESOURCE_ENDPOINTS from .monitoring.MonitoringLoops import MonitoringLoops @@ -117,11 +118,8 @@ def _raw_config_rules_to_grpc( errors.append(error_template.format(device_uuid, str(resource_key), str(resource_value))) continue - config_rule = device_config.config_rules.add() - config_rule.action = config_action - config_rule.custom.resource_key = resource_key - config_rule.custom.resource_value = \ - resource_value if isinstance(resource_value, str) else json.dumps(resource_value, sort_keys=True) + resource_value = json.loads(resource_value) if isinstance(resource_value, str) else resource_value + update_config_rule_custom(device_config.config_rules, resource_key, resource_value, new_action=config_action) return errors -- GitLab From c2a789ad113490eceae631d7e735358eebfc5055 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 09:07:56 +0000 Subject: [PATCH 23/31] Device component: - corrected parameters for composing config rules considering previous values - corrected unitary test for deconfigure --- src/device/service/Tools.py | 1 + src/device/tests/test_unitary_emulated.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/device/service/Tools.py b/src/device/service/Tools.py index 28b5391a7..970a35e37 100644 --- a/src/device/service/Tools.py +++ b/src/device/service/Tools.py @@ -119,6 +119,7 @@ def _raw_config_rules_to_grpc( continue resource_value = json.loads(resource_value) if isinstance(resource_value, str) else resource_value + resource_value = {field_name : (field_value, False) for field_name,field_value in resource_value.items()} update_config_rule_custom(device_config.config_rules, resource_key, resource_value, new_action=config_action) return errors diff --git a/src/device/tests/test_unitary_emulated.py b/src/device/tests/test_unitary_emulated.py index 1af1ad861..0e5f8e22d 100644 --- a/src/device/tests/test_unitary_emulated.py +++ b/src/device/tests/test_unitary_emulated.py @@ -379,7 +379,11 @@ def test_device_emulated_deconfigure( for config_rule in config_rules: assert config_rule.WhichOneof('config_rule') == 'custom' if config_rule.custom.resource_key.startswith('/endpoints/endpoint'): continue - config_rule_value = json.loads(config_rule.custom.resource_value) + if config_rule.custom.resource_key.startswith('_connect/'): continue + try: + config_rule_value = json.loads(config_rule.custom.resource_value) + except: # pylint: disable=bare-except + config_rule_value = config_rule.custom.resource_value if isinstance(config_rule_value, str) and config_rule_value.startswith('do_sampling (trigger:'): continue clean_config_rules.append(config_rule) LOGGER.info('clean_config_rules = {:s}'.format(str(clean_config_rules))) -- GitLab From 7ad1d40fd71c613948c9379363b21fed27caf807 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 09:08:48 +0000 Subject: [PATCH 24/31] Context component: - added logto config rule management for debug purposes --- src/context/service/database/ConfigRule.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index e35f246b6..0f0e9668c 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -14,6 +14,7 @@ import datetime, json, logging from sqlalchemy import delete +from sqlalchemy.dialects import postgresql from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from typing import Dict, List, Optional, Set @@ -104,8 +105,11 @@ def upsert_config_rules( str_config_rule = json.dumps(config_rule) raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) - #LOGGER.warning('uuids_to_delete={:s}'.format(str(uuids_to_delete))) - #LOGGER.warning('rules_to_upsert={:s}'.format(str(rules_to_upsert))) + LOGGER.warning('device_uuid={:s}'.format(str(device_uuid))) + LOGGER.warning('service_uuid={:s}'.format(str(service_uuid))) + LOGGER.warning('slice_uuid={:s}'.format(str(slice_uuid))) + LOGGER.warning('uuids_to_delete={:s}'.format(str(uuids_to_delete))) + LOGGER.warning('rules_to_upsert={:s}'.format(str(rules_to_upsert))) delete_affected = False upsert_affected = False @@ -116,7 +120,14 @@ def upsert_config_rules( if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid) if slice_uuid is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid == slice_uuid ) stmt = stmt.where(ConfigRuleModel.configrule_uuid.in_(uuids_to_delete)) - configrule_deletes = session.execute(stmt)#.fetchall() + + str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + LOGGER.warning('raw delete stmt={:s}'.format(str(str_stmt))) + + configrule_deletes = session.execute(stmt) + + LOGGER.warning('configrule_deletes.rowcount={:s}'.format(str(configrule_deletes.rowcount))) + delete_affected = int(configrule_deletes.rowcount) > 0 if len(rules_to_upsert) > 0: -- GitLab From 3aeb2e5cb6b1f47858fd743c4ea3342df59c96c6 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 10:03:08 +0000 Subject: [PATCH 25/31] Hackfest MockOSM: - updated default IP address - corrected error retrieval message --- hackfest/mock_osm/WimconnectorIETFL2VPN.py | 2 +- hackfest/mock_osm/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hackfest/mock_osm/WimconnectorIETFL2VPN.py b/hackfest/mock_osm/WimconnectorIETFL2VPN.py index e1273b4e4..aa4ca045f 100644 --- a/hackfest/mock_osm/WimconnectorIETFL2VPN.py +++ b/hackfest/mock_osm/WimconnectorIETFL2VPN.py @@ -73,7 +73,7 @@ class WimconnectorIETFL2VPN(SdnConnectorBase): response = requests.get(endpoint, auth=self.auth) http_code = response.status_code except requests.exceptions.RequestException as e: - raise SdnConnectorError(e.message, http_code=503) + raise SdnConnectorError(e.response, http_code=503) if http_code != 200: raise SdnConnectorError("Failed while authenticating", http_code=http_code) diff --git a/hackfest/mock_osm/__main__.py b/hackfest/mock_osm/__main__.py index e76616eab..410feda84 100644 --- a/hackfest/mock_osm/__main__.py +++ b/hackfest/mock_osm/__main__.py @@ -18,7 +18,7 @@ from .MockOSM import MockOSM LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -WIM_URL = 'http://10.0.2.15:80' +WIM_URL = 'http://10.0.2.10:80' WIM_USERNAME = 'admin' WIM_PASSWORD = 'admin' -- GitLab From 5ec2b2b6ee99c0401f6407cc75219b18348b0e70 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 10:03:21 +0000 Subject: [PATCH 26/31] MockOSM: - corrected error retrieval message --- src/tests/tools/mock_osm/WimconnectorIETFL2VPN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/tools/mock_osm/WimconnectorIETFL2VPN.py b/src/tests/tools/mock_osm/WimconnectorIETFL2VPN.py index e1273b4e4..aa4ca045f 100644 --- a/src/tests/tools/mock_osm/WimconnectorIETFL2VPN.py +++ b/src/tests/tools/mock_osm/WimconnectorIETFL2VPN.py @@ -73,7 +73,7 @@ class WimconnectorIETFL2VPN(SdnConnectorBase): response = requests.get(endpoint, auth=self.auth) http_code = response.status_code except requests.exceptions.RequestException as e: - raise SdnConnectorError(e.message, http_code=503) + raise SdnConnectorError(e.response, http_code=503) if http_code != 200: raise SdnConnectorError("Failed while authenticating", http_code=http_code) -- GitLab From 37da83e221ceaa473f4997d6f7ab175c4529eed1 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 15:14:01 +0000 Subject: [PATCH 27/31] Hackfest TAPI server: - updated methods to properly retrieve/store connectivity services --- .../tapi_connectivity_controller.py | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/hackfest/tapi/server/tapi_server/controllers/tapi_connectivity_controller.py b/hackfest/tapi/server/tapi_server/controllers/tapi_connectivity_controller.py index 640f5a72d..dc0118b7a 100644 --- a/hackfest/tapi/server/tapi_server/controllers/tapi_connectivity_controller.py +++ b/hackfest/tapi/server/tapi_server/controllers/tapi_connectivity_controller.py @@ -2559,8 +2559,42 @@ def data_tapi_commoncontext_tapi_connectivityconnectivity_context_post(body=None :rtype: None """ if connexion.request.is_json: - body = TapiConnectivityConnectivityContextWrapper.from_dict(connexion.request.get_json()) # noqa: E501 - return 'do some magic!' + #body = TapiConnectivityConnectivityContextWrapper.from_dict(connexion.request.get_json()) # noqa: E501 + raw_body = connexion.request.get_json() + if "tapi-connectivity:connectivity-service" in raw_body: + raw_body["connectivity-service"] = raw_body.pop("tapi-connectivity:connectivity-service") + if isinstance(raw_body["connectivity-service"], list) and len(raw_body["connectivity-service"]) > 0: + raw_body["connectivity-service"] = raw_body["connectivity-service"][0] + + connectivity_service = raw_body["connectivity-service"] + if "connectivity-constraint" in connectivity_service: + connectivity_constraint = connectivity_service.pop("connectivity-constraint") + if "requested-capacity" in connectivity_constraint: + connectivity_service["requested-capacity"] = connectivity_constraint.pop("requested-capacity") + if "connectivity-direction" in connectivity_constraint: + connectivity_service["connectivity-direction"] = connectivity_constraint.pop("connectivity-direction") + + body = TapiConnectivityConnectivityServiceWrapper.from_dict(raw_body) # noqa: E501 + + connection = TapiConnectivityConnection( + uuid=body.connectivity_service.uuid, + connection_end_point=[ + TapiConnectivityConnectionEndPointRef( + node_edge_point_uuid="node-1-port-3", connection_end_point_uuid="cep13"), + TapiConnectivityConnectionEndPointRef( + node_edge_point_uuid="node-3-port-2", connection_end_point_uuid="cep32"), + ] + ) + connection_ref = TapiConnectivityConnectionRef(connection.uuid) + body.connectivity_service.connection = [ connection_ref ] + + if database.context.connectivity_context is None: + database.context.connectivity_context = TapiConnectivityConnectivityContext( + connectivity_service=[], connection=[] + ) + + database.context.connectivity_context.connection.append(connection) + database.context.connectivity_context.connectivity_service.append(body.connectivity_service) def data_tapi_commoncontext_tapi_connectivityconnectivity_context_put(body=None): # noqa: E501 -- GitLab From 812f09fe91029ecc56f6e10b9cd482e13b06d183 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 15:21:10 +0000 Subject: [PATCH 28/31] Context component: - corrected ConstraintKinds in ConstraintModel - corrected UUID composition in ConfigRules to prevent collisions - corrected classification of config rules to upsert/delete - removed unneeded logs --- src/context/service/database/ConfigRule.py | 50 +-- src/context/service/database/Constraint.py | 167 ++++---- .../database/models/ConstraintModel.py | 364 +----------------- 3 files changed, 112 insertions(+), 469 deletions(-) diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 0f0e9668c..7e4a383c7 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -14,7 +14,7 @@ import datetime, json, logging from sqlalchemy import delete -from sqlalchemy.dialects import postgresql +#from sqlalchemy.dialects import postgresql from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from typing import Dict, List, Optional, Set @@ -44,29 +44,29 @@ def compose_config_rules_data( 'updated_at': now, } - parent_uuid = None + parent_kind,parent_uuid = '',None if device_uuid is not None: dict_config_rule['device_uuid'] = device_uuid - parent_uuid = device_uuid + parent_kind,parent_uuid = 'device',device_uuid elif service_uuid is not None: dict_config_rule['service_uuid'] = service_uuid - parent_uuid = service_uuid + parent_kind,parent_uuid = 'service',service_uuid elif slice_uuid is not None: dict_config_rule['slice_uuid'] = slice_uuid - parent_uuid = slice_uuid + parent_kind,parent_uuid = 'slice',slice_uuid else: MSG = 'Parent for ConfigRule({:s}) cannot be identified '+\ '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' str_config_rule = grpc_message_to_json_string(config_rule) raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) - + configrule_name = None if kind == ConfigRuleKindEnum.CUSTOM: - configrule_name = config_rule.custom.resource_key + configrule_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, config_rule.custom.resource_key) elif kind == ConfigRuleKindEnum.ACL: - endpoint_uuid = endpoint_get_uuid(config_rule.acl.endpoint_id, allow_random=False) + _, _, endpoint_uuid = endpoint_get_uuid(config_rule.acl.endpoint_id, allow_random=False) rule_set_name = config_rule.acl.rule_set.name - configrule_name = '{:s}/{:s}'.format(endpoint_uuid, rule_set_name) + configrule_name = '{:s}:{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid, rule_set_name) else: MSG = 'Name for ConfigRule({:s}) cannot be inferred '+\ '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' @@ -87,49 +87,39 @@ def upsert_config_rules( uuids_to_upsert : Dict[str, int] = dict() rules_to_upsert : List[Dict] = list() for config_rule in config_rules: - if config_rule['action'] == ORM_ConfigActionEnum.SET: - configrule_uuid = config_rule['configrule_uuid'] + configrule_uuid = config_rule['configrule_uuid'] + configrule_action = config_rule['action'] + if configrule_action == ORM_ConfigActionEnum.SET: position = uuids_to_upsert.get(configrule_uuid) if position is None: # if not added, add it rules_to_upsert.append(config_rule) - uuids_to_upsert[config_rule['configrule_uuid']] = len(rules_to_upsert) - 1 + uuids_to_upsert[configrule_uuid] = len(rules_to_upsert) - 1 else: # if already added, update occurrence rules_to_upsert[position] = config_rule - elif config_rule['action'] == ORM_ConfigActionEnum.DELETE: - uuids_to_delete.add(config_rule['configrule_uuid']) + elif configrule_action == ORM_ConfigActionEnum.DELETE: + uuids_to_delete.add(configrule_uuid) else: MSG = 'Action for ConfigRule({:s}) is not supported '+\ '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' str_config_rule = json.dumps(config_rule) raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid))) - LOGGER.warning('device_uuid={:s}'.format(str(device_uuid))) - LOGGER.warning('service_uuid={:s}'.format(str(service_uuid))) - LOGGER.warning('slice_uuid={:s}'.format(str(slice_uuid))) - LOGGER.warning('uuids_to_delete={:s}'.format(str(uuids_to_delete))) - LOGGER.warning('rules_to_upsert={:s}'.format(str(rules_to_upsert))) - delete_affected = False - upsert_affected = False - if len(uuids_to_delete) > 0: stmt = delete(ConfigRuleModel) if device_uuid is not None: stmt = stmt.where(ConfigRuleModel.device_uuid == device_uuid ) if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid) if slice_uuid is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid == slice_uuid ) stmt = stmt.where(ConfigRuleModel.configrule_uuid.in_(uuids_to_delete)) - - str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) - LOGGER.warning('raw delete stmt={:s}'.format(str(str_stmt))) - + #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt))) configrule_deletes = session.execute(stmt) - - LOGGER.warning('configrule_deletes.rowcount={:s}'.format(str(configrule_deletes.rowcount))) - + #LOGGER.warning('configrule_deletes.rowcount={:s}'.format(str(configrule_deletes.rowcount))) delete_affected = int(configrule_deletes.rowcount) > 0 + upsert_affected = False if len(rules_to_upsert) > 0: stmt = insert(ConfigRuleModel).values(rules_to_upsert) stmt = stmt.on_conflict_do_update( @@ -142,6 +132,8 @@ def upsert_config_rules( ) ) stmt = stmt.returning(ConfigRuleModel.created_at, ConfigRuleModel.updated_at) + #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) configrule_updates = session.execute(stmt).fetchall() upsert_affected = any([(updated_at > created_at) for created_at,updated_at in configrule_updates]) diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index 82629b25c..cf6e11ac9 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -14,13 +14,15 @@ import datetime, logging from sqlalchemy import delete +#from sqlalchemy.dialects import postgresql from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from typing import Dict, List, Optional from common.proto.context_pb2 import Constraint from common.tools.grpc.Tools import grpc_message_to_json_string from .models.ConstraintModel import ConstraintKindEnum, ConstraintModel -from .uuids._Builder import get_uuid_random +from .uuids._Builder import get_uuid_from_string +from .uuids.EndPoint import endpoint_get_uuid LOGGER = logging.getLogger(__name__) @@ -31,16 +33,50 @@ def compose_constraints_data( dict_constraints : List[Dict] = list() for position,constraint in enumerate(constraints): str_kind = constraint.WhichOneof('constraint') + kind = ConstraintKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_constraint = { - 'constraint_uuid': get_uuid_random(), - 'position' : position, - 'kind' : ConstraintKindEnum._member_map_.get(str_kind.upper()), # pylint: disable=no-member - 'data' : grpc_message_to_json_string(getattr(constraint, str_kind, {})), - 'created_at' : now, - 'updated_at' : now, + 'position' : position, + 'kind' : kind, + 'data' : grpc_message_to_json_string(getattr(constraint, str_kind, {})), + 'created_at': now, + 'updated_at': now, } - if service_uuid is not None: dict_constraint['service_uuid'] = service_uuid - if slice_uuid is not None: dict_constraint['slice_uuid' ] = slice_uuid + + parent_kind,parent_uuid = '',None + if service_uuid is not None: + dict_constraint['service_uuid'] = service_uuid + parent_kind,parent_uuid = 'service',service_uuid + elif slice_uuid is not None: + dict_constraint['slice_uuid'] = slice_uuid + parent_kind,parent_uuid = 'slice',slice_uuid + else: + MSG = 'Parent for Constraint({:s}) cannot be identified (service_uuid={:s}, slice_uuid={:s})' + str_constraint = grpc_message_to_json_string(constraint) + raise Exception(MSG.format(str_constraint, str(service_uuid), str(slice_uuid))) + + constraint_name = None + if kind == ConstraintKindEnum.CUSTOM: + constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, constraint.custom.constraint_type) + elif kind == ConstraintKindEnum.ENDPOINT_LOCATION: + _, _, endpoint_uuid = endpoint_get_uuid(constraint.endpoint_location.endpoint_id, allow_random=False) + location_kind = constraint.endpoint_location.location.WhichOneof('location') + constraint_name = '{:s}:{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid, location_kind) + elif kind == ConstraintKindEnum.ENDPOINT_PRIORITY: + _, _, endpoint_uuid = endpoint_get_uuid(constraint.endpoint_priority.endpoint_id, allow_random=False) + constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid) + elif kind in { + ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY, + ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION_LEVEL + }: + constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value) + else: + MSG = 'Name for Constraint({:s}) cannot be inferred (service_uuid={:s}, slice_uuid={:s})' + str_constraint = grpc_message_to_json_string(constraint) + raise Exception(MSG.format(str_constraint, str(service_uuid), str(slice_uuid))) + + constraint_uuid = get_uuid_from_string(constraint_name, prefix_for_name=parent_uuid) + dict_constraint['constraint_uuid'] = constraint_uuid + dict_constraints.append(dict_constraint) return dict_constraints @@ -48,84 +84,47 @@ def upsert_constraints( session : Session, constraints : List[Dict], service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None ) -> bool: - # TODO: do not delete all constraints; just add-remove as needed - stmt = delete(ConstraintModel) - if service_uuid is not None: stmt = stmt.where(ConstraintModel.service_uuid == service_uuid) - if slice_uuid is not None: stmt = stmt.where(ConstraintModel.slice_uuid == slice_uuid ) - session.execute(stmt) + uuids_to_upsert : Dict[str, int] = dict() + rules_to_upsert : List[Dict] = list() + for constraint in constraints: + constraint_uuid = constraint['constraint_uuid'] + position = uuids_to_upsert.get(constraint_uuid) + if position is None: + # if not added, add it + rules_to_upsert.append(constraint) + uuids_to_upsert[constraint_uuid] = len(rules_to_upsert) - 1 + else: + # if already added, update occurrence + rules_to_upsert[position] = constraint + + # Delete all constraints not in uuids_to_upsert + delete_affected = False + if len(uuids_to_upsert) > 0: + stmt = delete(ConstraintModel) + if service_uuid is not None: stmt = stmt.where(ConstraintModel.service_uuid == service_uuid) + if slice_uuid is not None: stmt = stmt.where(ConstraintModel.slice_uuid == slice_uuid ) + stmt = stmt.where(ConstraintModel.constraint_uuid.not_in(set(uuids_to_upsert.keys()))) + #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt))) + constraint_deletes = session.execute(stmt) + LOGGER.warning('constraint_deletes.rowcount={:s}'.format(str(constraint_deletes.rowcount))) + delete_affected = int(constraint_deletes.rowcount) > 0 - changed = False + upsert_affected = False if len(constraints) > 0: stmt = insert(ConstraintModel).values(constraints) - #stmt = stmt.on_conflict_do_update( - # index_elements=[ConstraintModel.configrule_uuid], - # set_=dict( - # updated_at = stmt.excluded.updated_at, - # ) - #) + stmt = stmt.on_conflict_do_update( + index_elements=[ConstraintModel.constraint_uuid], + set_=dict( + position = stmt.excluded.position, + data = stmt.excluded.data, + updated_at = stmt.excluded.updated_at, + ) + ) stmt = stmt.returning(ConstraintModel.created_at, ConstraintModel.updated_at) + #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) constraint_updates = session.execute(stmt).fetchall() - changed = any([(updated_at > created_at) for created_at,updated_at in constraint_updates]) + upsert_affected = any([(updated_at > created_at) for created_at,updated_at in constraint_updates]) - return changed - - -# def set_constraint(self, db_constraints: ConstraintsModel, grpc_constraint: Constraint, position: int -# ) -> Tuple[Union_ConstraintModel, bool]: -# with self.session() as session: -# -# grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint')) -# -# parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind) -# if parser is None: -# raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format( -# grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint))) -# -# # create specific constraint -# constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(grpc_constraint) -# str_constraint_id = str(uuid.uuid4()) -# LOGGER.info('str_constraint_id: {}'.format(str_constraint_id)) -# # str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id])) -# # str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':') -# -# # result : Tuple[Union_ConstraintModel, bool] = update_or_create_object( -# # database, constraint_class, str_constraint_key, constraint_data) -# constraint_data[constraint_class.main_pk_name()] = str_constraint_id -# db_new_constraint = constraint_class(**constraint_data) -# result: Tuple[Union_ConstraintModel, bool] = self.database.create_or_update(db_new_constraint) -# db_specific_constraint, updated = result -# -# # create generic constraint -# # constraint_fk_field_name = 'constraint_uuid'.format(constraint_kind.value) -# constraint_data = { -# 'constraints_uuid': db_constraints.constraints_uuid, 'position': position, 'kind': constraint_kind -# } -# -# db_new_constraint = ConstraintModel(**constraint_data) -# result: Tuple[Union_ConstraintModel, bool] = self.database.create_or_update(db_new_constraint) -# db_constraint, updated = result -# -# return db_constraint, updated -# -# def set_constraints(self, service_uuid: str, constraints_name : str, grpc_constraints -# ) -> List[Tuple[Union[ConstraintsModel, ConstraintModel], bool]]: -# with self.session() as session: -# # str_constraints_key = key_to_str([db_parent_pk, constraints_name], separator=':') -# # result : Tuple[ConstraintsModel, bool] = get_or_create_object(database, ConstraintsModel, str_constraints_key) -# result = session.query(ConstraintsModel).filter_by(constraints_uuid=service_uuid).one_or_none() -# created = None -# if result: -# created = True -# session.query(ConstraintsModel).filter_by(constraints_uuid=service_uuid).one_or_none() -# db_constraints = ConstraintsModel(constraints_uuid=service_uuid) -# session.add(db_constraints) -# -# db_objects = [(db_constraints, created)] -# -# for position,grpc_constraint in enumerate(grpc_constraints): -# result : Tuple[ConstraintModel, bool] = self.set_constraint( -# db_constraints, grpc_constraint, position) -# db_constraint, updated = result -# db_objects.append((db_constraint, updated)) -# -# return db_objects + return delete_affected or upsert_affected diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py index 51fc0b91d..e7767b4b2 100644 --- a/src/context/service/database/models/ConstraintModel.py +++ b/src/context/service/database/models/ConstraintModel.py @@ -20,11 +20,14 @@ from ._Base import _Base # Enum values should match name of field in Constraint message class ConstraintKindEnum(enum.Enum): - CUSTOM = 'custom' - ENDPOINT_LOCATION_REGION = 'ep_loc_region' - ENDPOINT_LOCATION_GPSPOSITION = 'ep_loc_gpspos' - ENDPOINT_PRIORITY = 'ep_priority' - SLA_AVAILABILITY = 'sla_avail' + CUSTOM = 'custom' + SCHEDULE = 'schedule' + ENDPOINT_LOCATION = 'endpoint_location' + ENDPOINT_PRIORITY = 'endpoint_priority' + SLA_CAPACITY = 'sla_capacity' + SLA_LATENCY = 'sla_latency' + SLA_AVAILABILITY = 'sla_availability' + SLA_ISOLATION_LEVEL = 'sla_isolation' class ConstraintModel(_Base): __tablename__ = 'constraint' @@ -46,354 +49,3 @@ class ConstraintModel(_Base): def dump(self) -> Dict: return {self.kind.value: json.loads(self.data)} - - -#import logging, operator -#from typing import Dict, List, Optional, Tuple, Type, Union -#from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object -#from common.orm.backend.Tools import key_to_str -#from common.proto.context_pb2 import Constraint -#from common.tools.grpc.Tools import grpc_message_to_json_string -#from .EndPointModel import EndPointModel -#from .Tools import fast_hasher -#from sqlalchemy import Column, ForeignKey, String, Float, CheckConstraint, Integer, Boolean, Enum -#from sqlalchemy.dialects.postgresql import UUID -#from context.service.database.models._Base import Base -#import enum -# -#LOGGER = logging.getLogger(__name__) -# -#def remove_dict_key(dictionary : Dict, key : str): -# dictionary.pop(key, None) -# return dictionary -# -#class ConstraintsModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'Constraints' -# constraints_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# -# @staticmethod -# def main_pk_name(): -# return 'constraints_uuid' -# -# -# def dump(self, constraints) -> List[Dict]: -# constraints = sorted(constraints, key=operator.itemgetter('position')) -# return [remove_dict_key(constraint, 'position') for constraint in constraints] -# -# -#class ConstraintCustomModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'ConstraintCustom' -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# constraint_type = Column(String, nullable=False) -# constraint_value = Column(String, nullable=False) -# -# @staticmethod -# def main_pk_name(): -# return 'constraint_uuid' -# -# -# def dump(self) -> Dict: # pylint: disable=arguments-differ -# return {'custom': {'constraint_type': self.constraint_type, 'constraint_value': self.constraint_value}} -# -# -#Union_ConstraintEndpoint = Union[ -# 'ConstraintEndpointLocationGpsPositionModel', 'ConstraintEndpointLocationRegionModel', -# 'ConstraintEndpointPriorityModel' -#] -# -#class ConstraintEndpointLocationRegionModel(Model): # pylint: disable=abstract-method -# endpoint_fk = ForeignKeyField(EndPointModel) -# region = StringField(required=True, allow_empty=False) -# -# def dump(self) -> Dict: # pylint: disable=arguments-differ -# json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id() -# return {'endpoint_location': {'endpoint_id': json_endpoint_id, 'location': {'region': self.region}}} -# -## def dump_endpoint_id(endpoint_constraint: Union_ConstraintEndpoint): -## db_endpoints_pks = list(endpoint_constraint.references(EndPointModel)) -## num_endpoints = len(db_endpoints_pks) -## if num_endpoints != 1: -## raise Exception('Wrong number({:d}) of associated Endpoints with constraint'.format(num_endpoints)) -## db_endpoint = EndPointModel(endpoint_constraint.database, db_endpoints_pks[0]) -## return db_endpoint.dump_id() -# -# -#class ConstraintEndpointLocationRegionModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'ConstraintEndpointLocationRegion' -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# endpoint_uuid = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid")) -# region = Column(String, nullable=False) -# -# @staticmethod -# def main_pk_name(): -# return 'constraint_uuid' -# -# def dump(self, endpoint) -> Dict: # pylint: disable=arguments-differ -# return {'endpoint_location': {'endpoint_id': endpoint.dump_id(), 'region': self.region}} -# -# def dump(self) -> Dict: # pylint: disable=arguments-differ -# gps_position = {'latitude': self.latitude, 'longitude': self.longitude} -# json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id() -# return {'endpoint_location': {'endpoint_id': json_endpoint_id, 'location': {'gps_position': gps_position}}} -# -#class ConstraintEndpointPriorityModel(Model): # pylint: disable=abstract-method -# endpoint_fk = ForeignKeyField(EndPointModel) -# priority = IntegerField(required=True, min_value=0) -# -# def dump(self) -> Dict: # pylint: disable=arguments-differ -# json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id() -# return {'endpoint_priority': {'endpoint_id': json_endpoint_id, 'priority': self.priority}} -# -#class ConstraintEndpointLocationGpsPositionModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'ConstraintEndpointLocationGpsPosition' -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# endpoint_uuid = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid")) -# latitude = Column(Float, CheckConstraint('latitude > -90.0 AND latitude < 90.0'), nullable=False) -# longitude = Column(Float, CheckConstraint('longitude > -90.0 AND longitude < 90.0'), nullable=False) -# -# def dump(self, endpoint) -> Dict: # pylint: disable=arguments-differ -# gps_position = {'latitude': self.latitude, 'longitude': self.longitude} -# return {'endpoint_location': {'endpoint_id': endpoint.dump_id(), 'gps_position': gps_position}} -# -# -#class ConstraintEndpointPriorityModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'ConstraintEndpointPriority' -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# endpoint_uuid = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid")) -# # endpoint_fk = ForeignKeyField(EndPointModel) -# # priority = FloatField(required=True) -# priority = Column(Float, nullable=False) -# @staticmethod -# def main_pk_name(): -# return 'constraint_uuid' -# -# def dump(self, endpoint) -> Dict: # pylint: disable=arguments-differ -# return {'endpoint_priority': {'endpoint_id': endpoint.dump_id(), 'priority': self.priority}} -# -# -#class ConstraintSlaAvailabilityModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'ConstraintSlaAvailability' -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# # num_disjoint_paths = IntegerField(required=True, min_value=1) -# num_disjoint_paths = Column(Integer, CheckConstraint('num_disjoint_paths > 1'), nullable=False) -# # all_active = BooleanField(required=True) -# all_active = Column(Boolean, nullable=False) -# @staticmethod -# def main_pk_name(): -# return 'constraint_uuid' -# -# def dump(self) -> Dict: # pylint: disable=arguments-differ -# return {'sla_availability': {'num_disjoint_paths': self.num_disjoint_paths, 'all_active': self.all_active}} -# -#Union_SpecificConstraint = Union[ -# ConstraintCustomModel, ConstraintEndpointLocationRegionModel, ConstraintEndpointLocationGpsPositionModel, -# ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel, -#] -# -#class ConstraintModel(Base): # pylint: disable=abstract-method -# __tablename__ = 'Constraint' -# # pk = PrimaryKeyField() -# # constraints_fk = ForeignKeyField(ConstraintsModel) -# constraint_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) -# constraints_uuid = Column(UUID(as_uuid=False), ForeignKey("Constraints.constraints_uuid"), primary_key=True) -# # kind = EnumeratedField(ConstraintKindEnum) -# kind = Column(Enum(ConstraintKindEnum, create_constraint=False, native_enum=False)) -# # position = IntegerField(min_value=0, required=True) -# position = Column(Integer, CheckConstraint('position >= 0'), nullable=False) -# # constraint_custom_fk = ForeignKeyField(ConstraintCustomModel, required=False) -# constraint_custom = Column(UUID(as_uuid=False), ForeignKey("ConstraintCustom.constraint_uuid")) -# # constraint_ep_loc_region_fk = ForeignKeyField(ConstraintEndpointLocationRegionModel, required=False) -# constraint_ep_loc_region = Column(UUID(as_uuid=False), ForeignKey("ConstraintEndpointLocationRegion.constraint_uuid")) -# # constraint_ep_loc_gpspos_fk = ForeignKeyField(ConstraintEndpointLocationGpsPositionModel, required=False) -# constraint_ep_loc_gpspos = Column(UUID(as_uuid=False), ForeignKey("ConstraintEndpointLocationGpsPosition.constraint_uuid")) -# # constraint_ep_priority_fk = ForeignKeyField(ConstraintEndpointPriorityModel, required=False) -# constraint_ep_priority = Column(UUID(as_uuid=False), ForeignKey("ConstraintEndpointPriority.constraint_uuid"),) -# # constraint_sla_avail_fk = ForeignKeyField(ConstraintSlaAvailabilityModel, required=False) -# constraint_sla_avail = Column(UUID(as_uuid=False), ForeignKey("ConstraintSlaAvailability.constraint_uuid")) -# -# @staticmethod -# def main_pk_name(): -# return 'constraint_uuid' -# -# # def delete(self) -> None: -# # field_name = 'constraint_{:s}_fk'.format(str(self.kind.value)) -# # specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None) -# # if specific_fk_value is None: -# # raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name)) -# # specific_fk_class = getattr(ConstraintModel, field_name, None) -# # foreign_model_class : Model = specific_fk_class.foreign_model -# # super().delete() -# # get_object(self.database, foreign_model_class, str(specific_fk_value)).delete() -# -# def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ -# field_name = 'constraint_{:s}'.format(str(self.kind.value)) -# specific_fk_value = getattr(self, field_name, None) -# if specific_fk_value is None: -# raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name)) -# specific_fk_class = getattr(ConstraintModel, field_name, None) -# foreign_model_class: Base = specific_fk_class.foreign_model -# constraint: Union_SpecificConstraint = get_object(self.database, foreign_model_class, str(specific_fk_value)) -# result = constraint.dump() -# if include_position: -# result['position'] = self.position -# return result -# -#Tuple_ConstraintSpecs = Tuple[Type, str, Dict, ConstraintKindEnum] -# -#def parse_constraint_custom(grpc_constraint) -> Tuple_ConstraintSpecs: -# constraint_class = ConstraintCustomModel -# str_constraint_id = grpc_constraint.custom.constraint_type -# constraint_data = { -# 'constraint_type' : grpc_constraint.custom.constraint_type, -# 'constraint_value': grpc_constraint.custom.constraint_value, -# } -# return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.CUSTOM -# -#def parse_constraint_endpoint_location(db_endpoint, grpc_constraint) -> Tuple_ConstraintSpecs: -# grpc_endpoint_id = grpc_constraint.endpoint_location.endpoint_id -# # str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id) -# -# str_constraint_id = db_endpoint.endpoint_uuid -# constraint_data = {'endpoint_fk': db_endpoint} -# -# grpc_location = grpc_constraint.endpoint_location.location -# location_kind = str(grpc_location.WhichOneof('location')) -# if location_kind == 'region': -# constraint_class = ConstraintEndpointLocationRegionModel -# constraint_data.update({'region': grpc_location.region}) -# return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_REGION -# elif location_kind == 'gps_position': -# constraint_class = ConstraintEndpointLocationGpsPositionModel -# gps_position = grpc_location.gps_position -# constraint_data.update({'latitude': gps_position.latitude, 'longitude': gps_position.longitude}) -# return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_GPSPOSITION -# else: -# MSG = 'Location kind {:s} in Constraint of kind endpoint_location is not implemented: {:s}' -# raise NotImplementedError(MSG.format(location_kind, grpc_message_to_json_string(grpc_constraint))) -# -#def parse_constraint_endpoint_priority(db_endpoint, grpc_constraint) -> Tuple_ConstraintSpecs: -# grpc_endpoint_id = grpc_constraint.endpoint_priority.endpoint_id -# # str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id) -# -# constraint_class = ConstraintEndpointPriorityModel -# str_constraint_id = db_endpoint.endpoint_uuid -# priority = grpc_constraint.endpoint_priority.priority -# constraint_data = {'endpoint_fk': db_endpoint, 'priority': priority} -# -# return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_PRIORITY -# -#def parse_constraint_sla_availability(grpc_constraint) -> Tuple_ConstraintSpecs: -# constraint_class = ConstraintSlaAvailabilityModel -# str_constraint_id = '' -# constraint_data = { -# 'num_disjoint_paths' : grpc_constraint.sla_availability.num_disjoint_paths, -# 'all_active': grpc_constraint.sla_availability.all_active, -# } -# return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.SLA_AVAILABILITY -# -#CONSTRAINT_PARSERS = { -# 'custom' : parse_constraint_custom, -# 'endpoint_location' : parse_constraint_endpoint_location, -# 'endpoint_priority' : parse_constraint_endpoint_priority, -# 'sla_availability' : parse_constraint_sla_availability, -#} -# -#Union_ConstraintModel = Union[ -# ConstraintCustomModel, ConstraintEndpointLocationGpsPositionModel, ConstraintEndpointLocationRegionModel, -# ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel -#] -# -## def set_constraint( -## db_constraints : ConstraintsModel, grpc_constraint : Constraint, position : int -## ) -> Tuple[Union_ConstraintModel, bool]: -## grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint')) -## -## parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind) -## if parser is None: -## raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format( -## grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint))) -## -## # create specific constraint -## constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(database, grpc_constraint) -## str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id])) -## str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':') -## result : Tuple[Union_ConstraintModel, bool] = update_or_create_object( -## database, constraint_class, str_constraint_key, constraint_data) -## db_specific_constraint, updated = result -## -## # create generic constraint -## constraint_fk_field_name = 'constraint_{:s}_fk'.format(constraint_kind.value) -## constraint_data = { -## 'constraints_fk': db_constraints, 'position': position, 'kind': constraint_kind, -## constraint_fk_field_name: db_specific_constraint -## } -## result : Tuple[ConstraintModel, bool] = update_or_create_object( -## database, ConstraintModel, str_constraint_key, constraint_data) -## db_constraint, updated = result -## -## return db_constraint, updated -## -## def set_constraints( -## database : Database, db_parent_pk : str, constraints_name : str, grpc_constraints -## ) -> List[Tuple[Union[ConstraintsModel, ConstraintModel], bool]]: -## -## str_constraints_key = key_to_str([db_parent_pk, constraints_name], separator=':') -## result : Tuple[ConstraintsModel, bool] = get_or_create_object(database, ConstraintsModel, str_constraints_key) -## db_constraints, created = result -## -## db_objects = [(db_constraints, created)] -## -## for position,grpc_constraint in enumerate(grpc_constraints): -## result : Tuple[ConstraintModel, bool] = set_constraint( -## database, db_constraints, grpc_constraint, position) -## db_constraint, updated = result -## db_objects.append((db_constraint, updated)) -## -## return db_objects -#def set_constraint( -# database : Database, db_constraints : ConstraintsModel, grpc_constraint : Constraint, position : int -#) -> Tuple[Union_ConstraintModel, bool]: -# grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint')) -# -# parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind) -# if parser is None: -# raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format( -# grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint))) -# -# # create specific constraint -# constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(database, grpc_constraint) -# str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id])) -# str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':') -# result : Tuple[Union_ConstraintModel, bool] = update_or_create_object( -# database, constraint_class, str_constraint_key, constraint_data) -# db_specific_constraint, updated = result -# -# # create generic constraint -# constraint_fk_field_name = 'constraint_{:s}_fk'.format(constraint_kind.value) -# constraint_data = { -# 'constraints_fk': db_constraints, 'position': position, 'kind': constraint_kind, -# constraint_fk_field_name: db_specific_constraint -# } -# result : Tuple[ConstraintModel, bool] = update_or_create_object( -# database, ConstraintModel, str_constraint_key, constraint_data) -# db_constraint, updated = result -# -# return db_constraint, updated -# -#def set_constraints( -# database : Database, db_parent_pk : str, constraints_name : str, grpc_constraints -#) -> List[Tuple[Union[ConstraintsModel, ConstraintModel], bool]]: -# -# str_constraints_key = key_to_str([constraints_name, db_parent_pk], separator=':') -# result : Tuple[ConstraintsModel, bool] = get_or_create_object(database, ConstraintsModel, str_constraints_key) -# db_constraints, created = result -# -# db_objects = [(db_constraints, created)] -# -# for position,grpc_constraint in enumerate(grpc_constraints): -# result : Tuple[ConstraintModel, bool] = set_constraint( -# database, db_constraints, grpc_constraint, position) -# db_constraint, updated = result -# db_objects.append((db_constraint, updated)) -# -# return db_objects -- GitLab From 7a612dd10d068f0fa17f38cb9b671b90d84239c7 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 15:23:26 +0000 Subject: [PATCH 29/31] Device component: - Added conditions to prevent reaching the driver when there is noting to configure/deconfigure --- src/device/service/Tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/device/service/Tools.py b/src/device/service/Tools.py index 970a35e37..9245e08a8 100644 --- a/src/device/service/Tools.py +++ b/src/device/service/Tools.py @@ -168,7 +168,7 @@ def compute_rules_to_add_delete( return resources_to_set, resources_to_delete def configure_rules(device : Device, driver : _Driver, resources_to_set : List[Tuple[str, Any]]) -> List[str]: - device_uuid = device.device_id.device_uuid.uuid + if len(resources_to_set) == 0: return [] results_setconfig = driver.SetConfig(resources_to_set) results_setconfig = [ @@ -176,11 +176,12 @@ def configure_rules(device : Device, driver : _Driver, resources_to_set : List[T for (resource_key, resource_value), result in zip(resources_to_set, results_setconfig) ] + device_uuid = device.device_id.device_uuid.uuid return _raw_config_rules_to_grpc( device_uuid, device.device_config, ERROR_SET, ConfigActionEnum.CONFIGACTION_SET, results_setconfig) def deconfigure_rules(device : Device, driver : _Driver, resources_to_delete : List[Tuple[str, Any]]) -> List[str]: - device_uuid = device.device_id.device_uuid.uuid + if len(resources_to_delete) == 0: return [] results_deleteconfig = driver.DeleteConfig(resources_to_delete) results_deleteconfig = [ @@ -188,6 +189,7 @@ def deconfigure_rules(device : Device, driver : _Driver, resources_to_delete : L for (resource_key, resource_value), result in zip(resources_to_delete, results_deleteconfig) ] + device_uuid = device.device_id.device_uuid.uuid return _raw_config_rules_to_grpc( device_uuid, device.device_config, ERROR_DELETE, ConfigActionEnum.CONFIGACTION_DELETE, results_deleteconfig) -- GitLab From 41befba7088c8db82ec3e4bfd7d2e80ad715cf1f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 15:24:10 +0000 Subject: [PATCH 30/31] Service component: - Added update of ServiceType during UpdateService --- src/service/service/ServiceServiceServicerImpl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 87eda2678..7d9ddeffb 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -16,7 +16,7 @@ import grpc, json, logging from typing import Optional from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.method_wrappers.ServiceExceptions import AlreadyExistsException, InvalidArgumentException -from common.proto.context_pb2 import Empty, Service, ServiceId, ServiceStatusEnum +from common.proto.context_pb2 import Empty, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum from common.proto.pathcomp_pb2 import PathCompRequest from common.proto.service_pb2_grpc import ServiceServiceServicer from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string @@ -93,6 +93,8 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): _service : Optional[Service] = get_service(context_client, request.service_id) service = Service() service.CopyFrom(request if _service is None else _service) + if service.service_type == ServiceTypeEnum.SERVICETYPE_UNKNOWN: # pylint: disable=no-member + service.service_type = request.service_type # pylint: disable=no-member service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED # pylint: disable=no-member del service.service_endpoint_ids[:] # pylint: disable=no-member -- GitLab From f6fe2ee80f94f39ab0bdd45d5afb04f49f88fbeb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sun, 29 Jan 2023 15:25:07 +0000 Subject: [PATCH 31/31] Slice component: - Corrected logic of Slice to take into account Context-generated UUIDs --- src/slice/service/SliceServiceServicerImpl.py | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index d693abd8f..413750662 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional import grpc, json, logging #, deepdiff from common.proto.context_pb2 import ( Empty, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum, Slice, SliceId, SliceStatusEnum) from common.proto.slice_pb2_grpc import SliceServiceServicer from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.tools.context_queries.InterDomain import is_multi_domain +from common.tools.context_queries.Slice import get_slice from common.tools.grpc.ConfigRules import copy_config_rules from common.tools.grpc.Constraints import copy_constraints from common.tools.grpc.EndPointIds import copy_endpoint_ids @@ -37,29 +39,32 @@ class SliceServiceServicerImpl(SliceServiceServicer): LOGGER.debug('Servicer Created') def create_update(self, request : Slice) -> SliceId: + # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is + # being modified. context_client = ContextClient() - try: - _slice = context_client.GetSlice(request.slice_id) - #json_current_slice = grpc_message_to_json(_slice) - except: # pylint: disable=bare-except - #json_current_slice = {} - slice_request = Slice() - slice_request.slice_id.CopyFrom(request.slice_id) # pylint: disable=no-member - slice_request.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member - context_client.SetSlice(slice_request) - _slice = context_client.GetSlice(request.slice_id) + slice_ro : Optional[Service] = get_slice( + context_client, request.slice_id.slice_uuid.uuid, request.slice_id.context_id.context_uuid.uuid, + rw_copy=False) + + slice_rw = Slice() + slice_rw.CopyFrom(request if slice_ro is None else slice_ro) + slice_rw.name = request.name + slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member + slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member - slice_request = Slice() - slice_request.CopyFrom(_slice) + copy_endpoint_ids(request.slice_endpoint_ids, slice_rw.slice_endpoint_ids ) # pylint: disable=no-member + copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member + copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member - if len(request.slice_endpoint_ids) < 2: + slice_id_with_uuids = context_client.SetSlice(slice_rw) + + if len(slice_rw.slice_endpoint_ids) < 2: # pylint: disable=no-member # unable to identify the kind of slice; just update endpoints, constraints and config rules # update the slice in database, and return # pylint: disable=no-member - copy_endpoint_ids(request.slice_endpoint_ids, slice_request.slice_endpoint_ids) - copy_constraints(request.slice_constraints, slice_request.slice_constraints) - copy_config_rules(request.slice_config.config_rules, slice_request.slice_config.config_rules) - return context_client.SetSlice(slice_request) + return context_client.SetSlice(slice_rw) + + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) @@ -67,9 +72,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): #changes = deepdiff.DeepDiff(json_current_slice, json_updated_slice) #LOGGER.info('changes = {:s}'.format(str(changes))) - if is_multi_domain(context_client, request.slice_endpoint_ids): + if is_multi_domain(context_client, slice_with_uuids.slice_endpoint_ids): interdomain_client = InterdomainClient() - slice_id = interdomain_client.RequestSlice(request) + slice_id = interdomain_client.RequestSlice(slice_with_uuids) slice_ = context_client.GetSlice(slice_id) slice_active = Slice() slice_active.CopyFrom(slice_) @@ -80,8 +85,8 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Local domain slice service_id = ServiceId() # pylint: disable=no-member - context_uuid = service_id.context_id.context_uuid.uuid = request.slice_id.context_id.context_uuid.uuid - service_uuid = service_id.service_uuid.uuid = request.slice_id.slice_uuid.uuid + context_uuid = service_id.context_id.context_uuid.uuid = slice_with_uuids.slice_id.context_id.context_uuid.uuid + service_uuid = service_id.service_uuid.uuid = slice_with_uuids.slice_id.slice_uuid.uuid service_client = ServiceClient() try: @@ -136,13 +141,13 @@ class SliceServiceServicerImpl(SliceServiceServicer): service_client.UpdateService(service_request) - copy_endpoint_ids(request.slice_endpoint_ids, slice_request.slice_endpoint_ids) - copy_constraints(request.slice_constraints, slice_request.slice_constraints) - copy_config_rules(request.slice_config.config_rules, slice_request.slice_config.config_rules) + #copy_endpoint_ids(request.slice_endpoint_ids, slice_with_uuids.slice_endpoint_ids) + #copy_constraints(request.slice_constraints, slice_with_uuids.slice_constraints) + #copy_config_rules(request.slice_config.config_rules, slice_with_uuids.slice_config.config_rules) - update_service_ids(slice_request.slice_service_ids, context_uuid, service_uuid) - context_client.SetSlice(slice_request) - slice_id = slice_request.slice_id + update_service_ids(slice_with_uuids.slice_service_ids, context_uuid, service_uuid) + context_client.SetSlice(slice_with_uuids) + slice_id = slice_with_uuids.slice_id slice_ = context_client.GetSlice(slice_id) slice_active = Slice() @@ -194,14 +199,14 @@ class SliceServiceServicerImpl(SliceServiceServicer): else: current_slice = Slice() current_slice.CopyFrom(_slice) - current_slice.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT + current_slice.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(current_slice) service_client = ServiceClient() for service_id in _slice.slice_service_ids: current_slice = Slice() - current_slice.slice_id.CopyFrom(_slice.slice_id) - slice_service_id = current_slice.slice_service_ids.add() + current_slice.slice_id.CopyFrom(_slice.slice_id) # pylint: disable=no-member + slice_service_id = current_slice.slice_service_ids.add() # pylint: disable=no-member slice_service_id.CopyFrom(service_id) context_client.UnsetSlice(current_slice) -- GitLab