From ccab8bcdde3afbf894bb02d45520cca0809cec46 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:05:08 +0000 Subject: [PATCH 1/6] Manifests - Context Service: - Added config flag ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY - Added config flag ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY --- manifests/contextservice.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 96735bf5f..df06c86b5 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -41,6 +41,10 @@ spec: value: "nats" - name: LOG_LEVEL value: "INFO" + - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY + value: "FALSE" + - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY + value: "FALSE" envFrom: - secretRef: name: crdb-data -- GitLab From ded39a9426ab2dbbde0506f23f983cb9c89419fb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:06:13 +0000 Subject: [PATCH 2/6] Common - Tests: - Extended MockServicerImpl_Context to add/remove topologies, services and slices to/from contexts --- src/common/tests/MockServicerImpl_Context.py | 63 ++++++++++++++++++-- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/common/tests/MockServicerImpl_Context.py b/src/common/tests/MockServicerImpl_Context.py index 55f87b7b0..837445b5a 100644 --- a/src/common/tests/MockServicerImpl_Context.py +++ b/src/common/tests/MockServicerImpl_Context.py @@ -143,17 +143,34 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetTopology(self, request: Topology, context : grpc.ServicerContext) -> TopologyId: LOGGER.debug('[SetTopology] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'topology[{:s}]'.format(str(request.topology_id.context_id.context_uuid.uuid)) + context_uuid = str(request.topology_id.context_id.context_uuid.uuid) + container_name = 'topology[{:s}]'.format(context_uuid) topology_uuid = request.topology_id.topology_uuid.uuid reply,_ = self._set(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _topology_id in context_.topology_ids: + if _topology_id.topology_uuid.uuid == topology_uuid: break + else: + # topology not found, add it + context_.topology_ids.add().topology_uuid.uuid = topology_uuid + LOGGER.debug('[SetTopology] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply def RemoveTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Empty: LOGGER.debug('[RemoveTopology] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'topology[{:s}]'.format(str(request.context_id.context_uuid.uuid)) + context_uuid = str(request.context_id.context_uuid.uuid) + container_name = 'topology[{:s}]'.format(context_uuid) topology_uuid = request.topology_uuid.uuid reply = self._del(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY, context) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _topology_id in context_.topology_ids: + if _topology_id.topology_uuid.uuid == topology_uuid: + context_.topology_ids.remove(_topology_id) + break + LOGGER.debug('[RemoveTopology] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply @@ -368,17 +385,34 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetSlice(self, request: Slice, context : grpc.ServicerContext) -> SliceId: LOGGER.debug('[SetSlice] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'slice[{:s}]'.format(str(request.slice_id.context_id.context_uuid.uuid)) + context_uuid = str(request.slice_id.context_id.context_uuid.uuid) + container_name = 'slice[{:s}]'.format(context_uuid) slice_uuid = request.slice_id.slice_uuid.uuid reply,_ = self._set(request, container_name, slice_uuid, 'slice_id', TOPIC_SLICE) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _slice_id in context_.slice_ids: + if _slice_id.slice_uuid.uuid == slice_uuid: break + else: + # slice not found, add it + context_.slice_ids.add().slice_uuid.uuid = slice_uuid + LOGGER.debug('[SetSlice] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply def RemoveSlice(self, request: SliceId, context : grpc.ServicerContext) -> Empty: LOGGER.debug('[RemoveSlice] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'slice[{:s}]'.format(str(request.context_id.context_uuid.uuid)) + context_uuid = str(request.slice_id.context_id.context_uuid.uuid) + container_name = 'slice[{:s}]'.format(context_uuid) slice_uuid = request.slice_uuid.uuid reply = self._del(request, container_name, slice_uuid, 'slice_id', TOPIC_SLICE, context) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _slice_id in context_.slice_ids: + if _slice_id.slice_uuid.uuid == slice_uuid: + context_.slice_ids.remove(_slice_id) + break + LOGGER.debug('[RemoveSlice] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply @@ -443,17 +477,34 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetService(self, request: Service, context : grpc.ServicerContext) -> ServiceId: LOGGER.debug('[SetService] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'service[{:s}]'.format(str(request.service_id.context_id.context_uuid.uuid)) + context_uuid = str(request.service_id.context_id.context_uuid.uuid) + container_name = 'service[{:s}]'.format(context_uuid) service_uuid = request.service_id.service_uuid.uuid reply,_ = self._set(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _service_id in context_.service_ids: + if _service_id.service_uuid.uuid == service_uuid: break + else: + # service not found, add it + context_.service_ids.add().service_uuid.uuid = service_uuid + LOGGER.debug('[SetService] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply def RemoveService(self, request: ServiceId, context : grpc.ServicerContext) -> Empty: LOGGER.debug('[RemoveService] request={:s}'.format(grpc_message_to_json_string(request))) - container_name = 'service[{:s}]'.format(str(request.context_id.context_uuid.uuid)) + context_uuid = str(request.service_id.context_id.context_uuid.uuid) + container_name = 'service[{:s}]'.format(context_uuid) service_uuid = request.service_uuid.uuid reply = self._del(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE, context) + + context_ = self.obj_db.get_entry('context', context_uuid, context) + for _service_id in context_.service_ids: + if _service_id.service_uuid.uuid == service_uuid: + context_.service_ids.remove(_service_id) + break + LOGGER.debug('[RemoveService] reply={:s}'.format(grpc_message_to_json_string(reply))) return reply -- GitLab From 8a452b4e51ee22c45fee4f132c79922e6f3a25f9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:06:41 +0000 Subject: [PATCH 3/6] Common - Tools - Context Queries: - Corrected return type in Topology methods --- src/common/tools/context_queries/Topology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tools/context_queries/Topology.py b/src/common/tools/context_queries/Topology.py index caf03ed0e..c90d59105 100644 --- a/src/common/tools/context_queries/Topology.py +++ b/src/common/tools/context_queries/Topology.py @@ -65,7 +65,7 @@ def get_topology( def get_topology_details( context_client : ContextClient, topology_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME, rw_copy : bool = False - ) -> Optional[Topology]: + ) -> Optional[TopologyDetails]: try: # pylint: disable=no-member topology_id = TopologyId() -- GitLab From c1ff5613d363a59103d17b3e1983148d567cfb58 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:07:09 +0000 Subject: [PATCH 4/6] Common - Tools - Descriptor: - Enabled explicit addition of devices and links to topologies --- src/common/tools/descriptor/Loader.py | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 916a73d30..c5468c19c 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -240,11 +240,16 @@ class DescriptorLoader: self._process_descr('slice', 'add', self.__ctx_cli.SetSlice, Slice, self.__slices ) self._process_descr('connection', 'add', self.__ctx_cli.SetConnection, Connection, self.__connections ) - # Update context and topology is useless: - # - devices and links are assigned to topologies automatically by Context component - # - topologies, services, and slices are assigned to contexts automatically by Context component + # By default the Context component automatically assigns devices and links to topologies based on their + # endpoints, and assigns topologies, services, and slices to contexts based on their identifiers. + + # The following statement is useless; up to now, any use case requires assigning a topology, service, or + # slice to a different context. #self._process_descr('context', 'update', self.__ctx_cli.SetContext, Context, self.__contexts ) - #self._process_descr('topology', 'update', self.__ctx_cli.SetTopology, Topology, self.__topologies ) + + # In some cases, it might be needed to assign devices and links to multiple topologies; the + # following statement performs that assignment. + self._process_descr('topology', 'update', self.__ctx_cli.SetTopology, Topology, self.__topologies ) #self.__ctx_cli.close() @@ -271,12 +276,17 @@ class DescriptorLoader: self._process_descr('service', 'update', self.__svc_cli.UpdateService, Service, self.__services ) self._process_descr('slice', 'add', self.__slc_cli.CreateSlice, Slice, self.__slices_add ) self._process_descr('slice', 'update', self.__slc_cli.UpdateSlice, Slice, self.__slices ) - - # Update context and topology is useless: - # - devices and links are assigned to topologies automatically by Context component - # - topologies, services, and slices are assigned to contexts automatically by Context component + + # By default the Context component automatically assigns devices and links to topologies based on their + # endpoints, and assigns topologies, services, and slices to contexts based on their identifiers. + + # The following statement is useless; up to now, any use case requires assigning a topology, service, or + # slice to a different context. #self._process_descr('context', 'update', self.__ctx_cli.SetContext, Context, self.__contexts ) - #self._process_descr('topology', 'update', self.__ctx_cli.SetTopology, Topology, self.__topologies ) + + # In some cases, it might be needed to assign devices and links to multiple topologies; the + # following statement performs that assignment. + self._process_descr('topology', 'update', self.__ctx_cli.SetTopology, Topology, self.__topologies ) #self.__slc_cli.close() #self.__svc_cli.close() -- GitLab From 82b47bf4a9bcb2b1f12a67d4b0568f2b7345dd06 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:08:01 +0000 Subject: [PATCH 5/6] Context component: - Added config flag ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY - Added config flag ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY - Added logic to explicitly add devices and links to topologies --- src/context/Config.py | 11 ++++ src/context/service/database/Topology.py | 67 ++++++++++++++++++++---- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/src/context/Config.py b/src/context/Config.py index 1549d9811..344b34a09 100644 --- a/src/context/Config.py +++ b/src/context/Config.py @@ -12,3 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from common.Settings import get_setting + +TRUE_VALUES = {'Y', 'YES', 'T', 'TRUE', 'E', 'ENABLE', 'ENABLED'} +def is_enabled(setting_name : str, default_value : bool) -> bool: + _is_enabled = get_setting(setting_name, default=None) + if _is_enabled is None: return default_value + str_is_enabled = str(_is_enabled).upper() + return str_is_enabled in TRUE_VALUES + +ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY', True) +ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY', True) diff --git a/src/context/service/database/Topology.py b/src/context/service/database/Topology.py index 1f0fb6c0b..e8f994155 100644 --- a/src/context/service/database/Topology.py +++ b/src/context/service/database/Topology.py @@ -17,17 +17,20 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker from sqlalchemy_cockroachdb import run_transaction -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Set from common.proto.context_pb2 import ( ContextId, Empty, EventTypeEnum, Topology, TopologyDetails, TopologyId, TopologyIdList, TopologyList) from common.message_broker.MessageBroker import MessageBroker from common.method_wrappers.ServiceExceptions import NotFoundException from common.tools.object_factory.Context import json_context_id from common.tools.object_factory.Topology import json_topology_id +from context.Config import ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY, ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY from .models.DeviceModel import DeviceModel from .models.LinkModel import LinkModel from .models.TopologyModel import TopologyDeviceModel, TopologyLinkModel, TopologyModel from .uuids.Context import context_get_uuid +from .uuids.Device import device_get_uuid +from .uuids.Link import link_get_uuid from .uuids.Topology import topology_get_uuid from .Events import notify_event_context, notify_event_topology @@ -94,15 +97,40 @@ def topology_set(db_engine : Engine, messagebroker : MessageBroker, request : To if len(topology_name) == 0: topology_name = request.topology_id.topology_uuid.uuid context_uuid,topology_uuid = topology_get_uuid(request.topology_id, topology_name=topology_name, allow_random=True) - # Ignore request.device_ids and request.link_ids. They are used for retrieving devices and links added into the - # topology. Explicit addition into the topology is done automatically when creating the devices and links, based - # on the topologies specified in the endpoints associated with the devices and links. + # By default, ignore request.device_ids and request.link_ids. They are used for retrieving + # devices and links added into the topology. Explicit addition into the topology is done + # automatically when creating the devices and links, based on the topologies specified in + # the endpoints associated with the devices and links. + # In some cases, it might be needed to add them explicitly; to allow that, activate flags + # ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY and/or ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY. - if len(request.device_ids) > 0: # pragma: no cover - LOGGER.warning('Items in field "device_ids" ignored. This field is used for retrieval purposes only.') + related_devices : List[Dict] = list() + if ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY: + device_uuids : Set[str] = set() + for device_id in request.device_ids: + device_uuid = device_get_uuid(device_id) + if device_uuid not in device_uuids: continue + related_devices.append({'topology_uuid': topology_uuid, 'device_uuid': device_uuid}) + device_uuids.add(device_uuid) + else: + if len(request.device_ids) > 0: # pragma: no cover + MSG = 'ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY={:s}; '.format(str(ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY)) + MSG += 'Items in field "device_ids" ignored. This field is used for retrieval purposes only.' + LOGGER.warning(MSG) - if len(request.link_ids) > 0: # pragma: no cover - LOGGER.warning('Items in field "link_ids" ignored. This field is used for retrieval purposes only.') + related_links : List[Dict] = list() + if ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY: + link_uuids : Set[str] = set() + for link_id in request.link_ids: + link_uuid = link_get_uuid(link_id) + if link_uuid not in link_uuids: continue + related_links.append({'topology_uuid': topology_uuid, 'link_uuid': link_uuid}) + link_uuids.add(link_uuid) + else: + if len(request.link_ids) > 0: # pragma: no cover + MSG = 'ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY={:s}; '.format(str(ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY)) + MSG += 'Items in field "link_ids" ignored. This field is used for retrieval purposes only.' + LOGGER.warning(MSG) now = datetime.datetime.utcnow() topology_data = [{ @@ -124,7 +152,28 @@ def topology_set(db_engine : Engine, messagebroker : MessageBroker, request : To ) stmt = stmt.returning(TopologyModel.created_at, TopologyModel.updated_at) created_at,updated_at = session.execute(stmt).fetchone() - return updated_at > created_at + + updated = updated_at > created_at + + updated_topology_device = False + if len(related_devices) > 0: + stmt = insert(TopologyDeviceModel).values(related_devices) + stmt = stmt.on_conflict_do_nothing( + index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid] + ) + topology_device_inserts = session.execute(stmt) + updated_topology_device = int(topology_device_inserts.rowcount) > 0 + + updated_topology_link = False + if len(related_links) > 0: + stmt = insert(TopologyLinkModel).values(related_links) + stmt = stmt.on_conflict_do_nothing( + index_elements=[TopologyLinkModel.topology_uuid, TopologyLinkModel.link_uuid] + ) + topology_link_inserts = session.execute(stmt) + updated_topology_link = int(topology_link_inserts.rowcount) > 0 + + return updated or updated_topology_device or updated_topology_link updated = run_transaction(sessionmaker(bind=db_engine), callback) context_id = json_context_id(context_uuid) -- GitLab From 7b4da382bb9a12eb7e68510d81fce32334689368 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 15 Dec 2023 16:10:10 +0000 Subject: [PATCH 6/6] Context component: - Changed default values to False for config flags ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY and ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY --- src/context/Config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/context/Config.py b/src/context/Config.py index 344b34a09..770b1f065 100644 --- a/src/context/Config.py +++ b/src/context/Config.py @@ -21,5 +21,6 @@ def is_enabled(setting_name : str, default_value : bool) -> bool: str_is_enabled = str(_is_enabled).upper() return str_is_enabled in TRUE_VALUES -ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY', True) -ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY', True) +DEFAULT_VALUE = False +ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY', DEFAULT_VALUE) +ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY = is_enabled('ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY', DEFAULT_VALUE) -- GitLab