From c47c372f107c6032da2ee1f01776481393704370 Mon Sep 17 00:00:00 2001 From: cmanso Date: Sun, 11 Dec 2022 15:47:46 +0100 Subject: [PATCH] Update scalability --- src/context/service/Database.py | 50 +++-- src/context/service/database/ConfigModel.py | 10 +- src/context/service/database/DeviceModel.py | 25 +-- src/context/service/database/EndPointModel.py | 2 +- src/context/service/database/LinkModel.py | 54 +++--- .../service/database/RelationModels.py | 103 ++++++----- src/context/service/database/ServiceModel.py | 28 ++- src/context/service/database/TopologyModel.py | 21 +-- .../grpc_server/ContextServiceServicerImpl.py | 171 ++++++++++++------ src/context/tests/Objects.py | 35 ++-- 10 files changed, 271 insertions(+), 228 deletions(-) diff --git a/src/context/service/Database.py b/src/context/service/Database.py index bf970b356..2b699203a 100644 --- a/src/context/service/Database.py +++ b/src/context/service/Database.py @@ -16,6 +16,9 @@ class Database(Session): super().__init__() self.session = session + def get_session(self): + return self.session + def get_all(self, model): result = [] with self.session() as session: @@ -27,22 +30,21 @@ class Database(Session): def create_or_update(self, model): with self.session() as session: att = getattr(model, model.main_pk_name()) - obj = self.get_object(type(model), att) - filt = {model.main_pk_name(): att} t_model = type(model) - found = session.query(t_model).filter_by(**filt).one_or_none() - if found: + obj = session.query(t_model).filter_by(**filt).one_or_none() + + if obj: + for key in obj.__table__.columns.keys(): + setattr(obj, key, getattr(model, key)) found = True + session.commit() + return obj, found else: found = False - - session.merge(model) - session.commit() - - obj = self.get_object(t_model, att) - - return model, found + session.add(model) + session.commit() + return model, found def create(self, model): with self.session() as session: @@ -85,7 +87,6 @@ class Database(Session): for table in meta.sorted_tables: for row in engine.execute(table.select()): result.append((table.name, dict(row))) - LOGGER.info(result) return result @@ -98,10 +99,27 @@ class Database(Session): if raise_if_not_found: raise NotFoundException(model_class.__name__.replace('Model', ''), main_key) - return get + dump = None + if hasattr(get, 'dump'): + dump = get.dump() + return get, dump + + def get_object_filter(self, model_class: Base, filt, raise_if_not_found=False): + with self.session() as session: + get = session.query(model_class).filter_by(**filt).all() + + if not get: + if raise_if_not_found: + raise NotFoundException(model_class.__name__.replace('Model', '')) + else: + return None, None + + if isinstance(get, list): + return get, [obj.dump() for obj in get] + + return get, get.dump() - def get_or_create(self, model_class: Base, key_parts: List[str], filt=None) -> Tuple[Base, bool]: - str_key = key_to_str(key_parts) + def get_or_create(self, model_class: Base, key_parts: str, filt=None) -> Tuple[Base, bool]: if not filt: filt = {model_class.main_pk_name(): key_parts} with self.session() as session: @@ -110,7 +128,7 @@ class Database(Session): return get, False else: obj = model_class() - setattr(obj, model_class.main_pk_name(), str_key) + setattr(obj, model_class.main_pk_name(), key_parts) session.add(obj) session.commit() return obj, True diff --git a/src/context/service/database/ConfigModel.py b/src/context/service/database/ConfigModel.py index 40069185f..2ec22985c 100644 --- a/src/context/service/database/ConfigModel.py +++ b/src/context/service/database/ConfigModel.py @@ -40,13 +40,7 @@ class ConfigModel(Base): # pylint: disable=abstract-method config_uuid = Column(UUID(as_uuid=False), primary_key=True) # Relationships - config_rule = relationship("ConfigRuleModel", back_populates="config", lazy='joined') - - - def delete(self) -> None: - db_config_rule_pks = self.references(ConfigRuleModel) - for pk,_ in db_config_rule_pks: ConfigRuleModel(self.database, pk).delete() - super().delete() + config_rule = relationship("ConfigRuleModel", cascade="all,delete", back_populates="config", lazy='joined') def dump(self) -> List[Dict]: config_rules = [] @@ -75,7 +69,7 @@ class ConfigRuleModel(Base): # pylint: disable=abstract-method ) # Relationships - config = relationship("ConfigModel", back_populates="config_rule") + config = relationship("ConfigModel", passive_deletes=True, back_populates="config_rule") def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ result = { diff --git a/src/context/service/database/DeviceModel.py b/src/context/service/database/DeviceModel.py index 122da50af..b7e7efed4 100644 --- a/src/context/service/database/DeviceModel.py +++ b/src/context/service/database/DeviceModel.py @@ -54,30 +54,10 @@ class DeviceModel(Base): native_enum=False)) # Relationships - device_config = relationship("ConfigModel", passive_deletes="all, delete", lazy="joined") + device_config = relationship("ConfigModel", passive_deletes=True, lazy="joined") driver = relationship("DriverModel", passive_deletes=True, back_populates="device") endpoints = relationship("EndPointModel", passive_deletes=True, back_populates="device") - # topology = relationship("TopologyModel", lazy="joined") - - # def delete(self) -> None: - # # pylint: disable=import-outside-toplevel - # from .EndPointModel import EndPointModel - # from .RelationModels import TopologyDeviceModel - # - # for db_endpoint_pk,_ in self.references(EndPointModel): - # EndPointModel(self.database, db_endpoint_pk).delete() - # - # for db_topology_device_pk,_ in self.references(TopologyDeviceModel): - # TopologyDeviceModel(self.database, db_topology_device_pk).delete() - # - # for db_driver_pk,_ in self.references(DriverModel): - # DriverModel(self.database, db_driver_pk).delete() - # - # super().delete() - # - # ConfigModel(self.database, self.device_config_fk).delete() - def dump_id(self) -> Dict: return {'device_uuid': {'uuid': self.device_uuid}} @@ -86,10 +66,7 @@ class DeviceModel(Base): def dump_drivers(self) -> List[int]: response = [] - for a in self.driver: - LOGGER.info('DUMPPPPPPPPPPPPPPPPPPPPPIIIIIIIIIIIIIIIIIIIIIIINNNNNNNNNNNNNNNGGGGGGGGGGGGGGGGGGg') - LOGGER.info('aasdfadsf: {}'.format(a.dump())) response.append(a.dump()) return response diff --git a/src/context/service/database/EndPointModel.py b/src/context/service/database/EndPointModel.py index a4381a2e3..fb2c9d26a 100644 --- a/src/context/service/database/EndPointModel.py +++ b/src/context/service/database/EndPointModel.py @@ -20,7 +20,7 @@ from common.orm.backend.Tools import key_to_str from common.proto.context_pb2 import EndPointId from .KpiSampleType import ORM_KpiSampleTypeEnum, grpc_to_enum__kpi_sample_type from sqlalchemy import Column, ForeignKey, String, Enum, ForeignKeyConstraint -from sqlalchemy.dialects.postgresql import UUID, ARRAY +from sqlalchemy.dialects.postgresql import UUID from context.service.database.Base import Base from sqlalchemy.orm import relationship LOGGER = logging.getLogger(__name__) diff --git a/src/context/service/database/LinkModel.py b/src/context/service/database/LinkModel.py index 8f1d971c3..025709dfd 100644 --- a/src/context/service/database/LinkModel.py +++ b/src/context/service/database/LinkModel.py @@ -14,39 +14,39 @@ import logging, operator from typing import Dict, List -from common.orm.fields.PrimaryKeyField import PrimaryKeyField -from common.orm.fields.StringField import StringField -from common.orm.model.Model import Model -from common.orm.HighLevel import get_related_objects +from sqlalchemy import Column, ForeignKey +from sqlalchemy.dialects.postgresql import UUID +from context.service.database.Base import Base +from sqlalchemy.orm import relationship LOGGER = logging.getLogger(__name__) -class LinkModel(Model): - pk = PrimaryKeyField() - link_uuid = StringField(required=True, allow_empty=False) +class LinkModel(Base): + __tablename__ = 'Link' + link_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) - def delete(self) -> None: - #pylint: disable=import-outside-toplevel - from .RelationModels import LinkEndPointModel, TopologyLinkModel - - for db_link_endpoint_pk,_ in self.references(LinkEndPointModel): - LinkEndPointModel(self.database, db_link_endpoint_pk).delete() - - for db_topology_link_pk,_ in self.references(TopologyLinkModel): - TopologyLinkModel(self.database, db_topology_link_pk).delete() - - super().delete() + @staticmethod + def main_pk_name(): + return 'link_uuid' def dump_id(self) -> Dict: return {'link_uuid': {'uuid': self.link_uuid}} def dump_endpoint_ids(self) -> List[Dict]: - from .RelationModels import LinkEndPointModel # pylint: disable=import-outside-toplevel - db_endpoints = get_related_objects(self, LinkEndPointModel, 'endpoint_fk') - return [db_endpoint.dump_id() for db_endpoint in sorted(db_endpoints, key=operator.attrgetter('pk'))] - - def dump(self) -> Dict: - return { - 'link_id': self.dump_id(), - 'link_endpoint_ids': self.dump_endpoint_ids(), - } + return [endpoint.dump_id() for endpoint in self.endpoints] + + def dump(self, endpoints=None) -> Dict: + result = { + 'link_id': self.dump_id() + } + if endpoints: + result['link_endpoint_ids'] = [] + for endpoint in endpoints: + dump = endpoint.dump_id() + LOGGER.info(dump) + result['link_endpoint_ids'].append(dump) + + LOGGER.info(result['link_endpoint_ids']) + + LOGGER.info(result) + return result diff --git a/src/context/service/database/RelationModels.py b/src/context/service/database/RelationModels.py index 98b077a77..e69feadc4 100644 --- a/src/context/service/database/RelationModels.py +++ b/src/context/service/database/RelationModels.py @@ -13,55 +13,68 @@ # limitations under the License. import logging -from common.orm.fields.ForeignKeyField import ForeignKeyField -from common.orm.fields.PrimaryKeyField import PrimaryKeyField -from common.orm.model.Model import Model -from .ConnectionModel import ConnectionModel -from .DeviceModel import DeviceModel -from .EndPointModel import EndPointModel -from .LinkModel import LinkModel -from .ServiceModel import ServiceModel -from .SliceModel import SliceModel -from .TopologyModel import TopologyModel +from sqlalchemy import Column, ForeignKey +from sqlalchemy.dialects.postgresql import UUID +from context.service.database.Base import Base LOGGER = logging.getLogger(__name__) +# +# class ConnectionSubServiceModel(Model): # pylint: disable=abstract-method +# pk = PrimaryKeyField() +# connection_fk = ForeignKeyField(ConnectionModel) +# sub_service_fk = ForeignKeyField(ServiceModel) +# +class LinkEndPointModel(Base): # pylint: disable=abstract-method + __tablename__ = 'LinkEndPoint' + # uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) + link_uuid = Column(UUID(as_uuid=False), ForeignKey("Link.link_uuid")) + endpoint_uuid = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid"), primary_key=True) -class ConnectionSubServiceModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - connection_fk = ForeignKeyField(ConnectionModel) - sub_service_fk = ForeignKeyField(ServiceModel) - -class LinkEndPointModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - link_fk = ForeignKeyField(LinkModel) - endpoint_fk = ForeignKeyField(EndPointModel) - -class ServiceEndPointModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - service_fk = ForeignKeyField(ServiceModel) - endpoint_fk = ForeignKeyField(EndPointModel) - -class SliceEndPointModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - slice_fk = ForeignKeyField(SliceModel) - endpoint_fk = ForeignKeyField(EndPointModel) + @staticmethod + def main_pk_name(): + return 'endpoint_uuid' -class SliceServiceModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - slice_fk = ForeignKeyField(SliceModel) - service_fk = ForeignKeyField(ServiceModel) +# +# class ServiceEndPointModel(Model): # pylint: disable=abstract-method +# pk = PrimaryKeyField() +# service_fk = ForeignKeyField(ServiceModel) +# endpoint_fk = ForeignKeyField(EndPointModel) +# +# class SliceEndPointModel(Model): # pylint: disable=abstract-method +# pk = PrimaryKeyField() +# slice_fk = ForeignKeyField(SliceModel) +# endpoint_fk = ForeignKeyField(EndPointModel) +# +# class SliceServiceModel(Model): # pylint: disable=abstract-method +# pk = PrimaryKeyField() +# slice_fk = ForeignKeyField(SliceModel) +# service_fk = ForeignKeyField(ServiceMo# pylint: disable=abstract-method +# __tablename__ = 'LinkEndPoint' +# uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) +# link_uuid = Column(UUID(as_uuid=False), ForeignKey("Link.link_uuid")) +# endpoint_uuid = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid")) +#del) +# +# class SliceSubSliceModel(Model): # pylint: disable=abstract-method +# pk = PrimaryKeyField() +# slice_fk = ForeignKeyField(SliceModel) +# sub_slice_fk = ForeignKeyField(SliceModel) -class SliceSubSliceModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - slice_fk = ForeignKeyField(SliceModel) - sub_slice_fk = ForeignKeyField(SliceModel) +class TopologyDeviceModel(Base): # pylint: disable=abstract-method + __tablename__ = 'TopologyDevice' + # uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) + topology_uuid = Column(UUID(as_uuid=False), ForeignKey("Topology.topology_uuid")) + device_uuid = Column(UUID(as_uuid=False), ForeignKey("Device.device_uuid"), primary_key=True) -class TopologyDeviceModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - topology_fk = ForeignKeyField(TopologyModel) - device_fk = ForeignKeyField(DeviceModel) + @staticmethod + def main_pk_name(): + return 'device_uuid' +# +class TopologyLinkModel(Base): # pylint: disable=abstract-method + __tablename__ = 'TopologyLink' + topology_uuid = Column(UUID(as_uuid=False), ForeignKey("Topology.topology_uuid")) + link_uuid = Column(UUID(as_uuid=False), ForeignKey("Link.link_uuid"), primary_key=True) -class TopologyLinkModel(Model): # pylint: disable=abstract-method - pk = PrimaryKeyField() - topology_fk = ForeignKeyField(TopologyModel) - link_fk = ForeignKeyField(LinkModel) + @staticmethod + def main_pk_name(): + return 'link_uuid' \ No newline at end of file diff --git a/src/context/service/database/ServiceModel.py b/src/context/service/database/ServiceModel.py index 8b32d1cc9..a5223d615 100644 --- a/src/context/service/database/ServiceModel.py +++ b/src/context/service/database/ServiceModel.py @@ -13,20 +13,17 @@ # limitations under the License. import functools, logging, operator -from enum import Enum +from sqlalchemy import Column, ForeignKey, String, Enum from typing import Dict, List -from common.orm.fields.EnumeratedField import EnumeratedField -from common.orm.fields.ForeignKeyField import ForeignKeyField -from common.orm.fields.PrimaryKeyField import PrimaryKeyField -from common.orm.fields.StringField import StringField -from common.orm.model.Model import Model from common.orm.HighLevel import get_related_objects from common.proto.context_pb2 import ServiceStatusEnum, ServiceTypeEnum from .ConfigModel import ConfigModel from .ConstraintModel import ConstraintsModel from .ContextModel import ContextModel from .Tools import grpc_to_enum - +from sqlalchemy import Column, ForeignKey +from sqlalchemy.dialects.postgresql import UUID +from context.service.database.Base import Base LOGGER = logging.getLogger(__name__) class ORM_ServiceTypeEnum(Enum): @@ -47,14 +44,15 @@ class ORM_ServiceStatusEnum(Enum): grpc_to_enum__service_status = functools.partial( grpc_to_enum, ServiceStatusEnum, ORM_ServiceStatusEnum) -class ServiceModel(Model): - pk = PrimaryKeyField() - context_fk = ForeignKeyField(ContextModel) - service_uuid = StringField(required=True, allow_empty=False) - service_type = EnumeratedField(ORM_ServiceTypeEnum, required=True) - service_constraints_fk = ForeignKeyField(ConstraintsModel) - service_status = EnumeratedField(ORM_ServiceStatusEnum, required=True) - service_config_fk = ForeignKeyField(ConfigModel) +class ServiceModel(Base): + __tablename__ = 'Service' + + service_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) + service_type = Column(Enum(ORM_ServiceTypeEnum, create_constraint=False, native_enum=False, allow_empty=False)) + # service_constraints = Column(UUID(as_uuid=False), ForeignKey("EndPoint.endpoint_uuid", ondelete='SET NULL')) + # context_fk = ForeignKeyField(ContextModel) + service_status = Column(Enum(ORM_ServiceStatusEnum, create_constraint=False, native_enum=False, allow_empty=False)) + # service_config_fk = ForeignKeyField(ConfigModel) def delete(self) -> None: #pylint: disable=import-outside-toplevel diff --git a/src/context/service/database/TopologyModel.py b/src/context/service/database/TopologyModel.py index 2925a27fa..063a1f511 100644 --- a/src/context/service/database/TopologyModel.py +++ b/src/context/service/database/TopologyModel.py @@ -26,7 +26,7 @@ class TopologyModel(Base): topology_uuid = Column(UUID(as_uuid=False), primary_key=True, unique=True) # Relationships - context = relationship("ContextModel", back_populates="topology", lazy="joined") + context = relationship("ContextModel", back_populates="topology") def dump_id(self) -> Dict: context_id = self.context.dump_id() @@ -39,21 +39,12 @@ class TopologyModel(Base): def main_pk_name() -> str: return 'topology_uuid' - """def dump_device_ids(self) -> List[Dict]: - from .RelationModels import TopologyDeviceModel # pylint: disable=import-outside-toplevel - db_devices = get_related_objects(self, TopologyDeviceModel, 'device_fk') - return [db_device.dump_id() for db_device in sorted(db_devices, key=operator.attrgetter('pk'))] - - def dump_link_ids(self) -> List[Dict]: - from .RelationModels import TopologyLinkModel # pylint: disable=import-outside-toplevel - db_links = get_related_objects(self, TopologyLinkModel, 'link_fk') - return [db_link.dump_id() for db_link in sorted(db_links, key=operator.attrgetter('pk'))] - """ - def dump( # pylint: disable=arguments-differ - self, include_devices=True, include_links=True + self, devices=None, links=None ) -> Dict: result = {'topology_id': self.dump_id()} - # if include_devices: result['device_ids'] = self.dump_device_ids() - # if include_links: result['link_ids'] = self.dump_link_ids() + if devices: + result['device_ids'] = [device.dump_id() for device in devices] + if links: + result['link_ids'] = [link.dump_id() for link in links] return result diff --git a/src/context/service/grpc_server/ContextServiceServicerImpl.py b/src/context/service/grpc_server/ContextServiceServicerImpl.py index 108ab9950..264ae3198 100644 --- a/src/context/service/grpc_server/ContextServiceServicerImpl.py +++ b/src/context/service/grpc_server/ContextServiceServicerImpl.py @@ -60,6 +60,7 @@ from context.service.database.Events import notify_event from context.service.database.EndPointModel import EndPointModel from context.service.database.EndPointModel import KpiSampleTypeModel from context.service.database.LinkModel import LinkModel +from context.service.database.RelationModels import (TopologyDeviceModel, TopologyLinkModel, LinkEndPointModel) from .Constants import ( CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE, @@ -202,16 +203,30 @@ class ContextServiceServicerImpl(ContextServiceServicer): @safe_and_metered_rpc_method(METRICS, LOGGER) def GetTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Topology: - context_uuid = request.context_id.context_uuid.uuid topology_uuid = request.topology_uuid.uuid + result, dump = self.database.get_object(TopologyModel, topology_uuid, True) with self.session() as session: - result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).options(contains_eager(TopologyModel.context)).one_or_none() + devs = None + links = None - if not result: - raise NotFoundException(TopologyModel.__name__.replace('Model', ''), topology_uuid) + filt = {'topology_uuid': topology_uuid} + topology_devices = session.query(TopologyDeviceModel).filter_by(**filt).all() + if topology_devices: + devs = [] + for td in topology_devices: + filt = {'device_uuid': td.device_uuid} + devs.append(session.query(DeviceModel).filter_by(**filt).one()) + + filt = {'topology_uuid': topology_uuid} + topology_links = session.query(TopologyLinkModel).filter_by(**filt).all() + if topology_links: + links = [] + for tl in topology_links: + filt = {'link_uuid': tl.link_uuid} + links.append(session.query(LinkModel).filter_by(**filt).one()) - return Topology(**result.dump()) + return Topology(**result.dump(devs, links)) @safe_and_metered_rpc_method(METRICS, LOGGER) @@ -221,15 +236,30 @@ class ContextServiceServicerImpl(ContextServiceServicer): with self.session() as session: topology_add = TopologyModel(topology_uuid=topology_uuid, context_uuid=context_uuid) updated = True - result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).one_or_none() - if not result: + db_topology = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).one_or_none() + if not db_topology: updated = False session.merge(topology_add) session.commit() - result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).one_or_none() + db_topology = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).one_or_none() + + for device_id in request.device_ids: + device_uuid = device_id.device_uuid.uuid + td = TopologyDeviceModel(topology_uuid=topology_uuid, device_uuid=device_uuid) + result: Tuple[TopologyDeviceModel, bool] = self.database.create_or_update(td) + + + for link_id in request.link_ids: + link_uuid = link_id.link_uuid.uuid + db_link = session.query(LinkModel).filter( + LinkModel.link_uuid == link_uuid).one_or_none() + tl = TopologyLinkModel(topology_uuid=topology_uuid, link_uuid=link_uuid) + result: Tuple[TopologyDeviceModel, bool] = self.database.create_or_update(tl) + + event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE - dict_topology_id = result.dump_id() + dict_topology_id = db_topology.dump_id() notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': dict_topology_id}) return TopologyId(**dict_topology_id) @@ -289,9 +319,10 @@ class ContextServiceServicerImpl(ContextServiceServicer): with self.session() as session: device_uuid = request.device_id.device_uuid.uuid - for i,endpoint in enumerate(request.device_endpoints): + for i, endpoint in enumerate(request.device_endpoints): endpoint_device_uuid = endpoint.endpoint_id.device_id.device_uuid.uuid - if len(endpoint_device_uuid) == 0: endpoint_device_uuid = device_uuid + if len(endpoint_device_uuid) == 0: + endpoint_device_uuid = device_uuid if device_uuid != endpoint_device_uuid: raise InvalidArgumentException( 'request.device_endpoints[{:d}].device_id.device_uuid.uuid'.format(i), endpoint_device_uuid, @@ -313,12 +344,12 @@ class ContextServiceServicerImpl(ContextServiceServicer): self.set_drivers(db_device, request.device_drivers) - for i,endpoint in enumerate(request.device_endpoints): + for i, endpoint in enumerate(request.device_endpoints): endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid - endpoint_device_uuid = endpoint.endpoint_id.device_id.device_uuid.uuid - if len(endpoint_device_uuid) == 0: endpoint_device_uuid = device_uuid + # endpoint_device_uuid = endpoint.endpoint_id.device_id.device_uuid.uuid + # if len(endpoint_device_uuid) == 0: + # endpoint_device_uuid = device_uuid - str_endpoint_key = key_to_str([device_uuid, endpoint_uuid]) endpoint_attributes = { 'device_uuid' : db_device.device_uuid, 'endpoint_uuid': endpoint_uuid, @@ -328,17 +359,19 @@ class ContextServiceServicerImpl(ContextServiceServicer): endpoint_topology_context_uuid = endpoint.endpoint_id.topology_id.context_id.context_uuid.uuid endpoint_topology_uuid = endpoint.endpoint_id.topology_id.topology_uuid.uuid if len(endpoint_topology_context_uuid) > 0 and len(endpoint_topology_uuid) > 0: - str_topology_key = key_to_str([endpoint_topology_context_uuid, endpoint_topology_uuid]) + # str_topology_key = key_to_str([endpoint_topology_context_uuid, endpoint_topology_uuid]) - db_topology: TopologyModel = self.database.get_object(TopologyModel, endpoint_topology_uuid) - new_topo = TopologyModel(context_uuid=db_topology.context_uuid, topology_uuid=db_topology.topology_uuid, device_uuids=db_device.device_uuid) + db_topology, topo_dump = self.database.get_object(TopologyModel, endpoint_topology_uuid) - self.database.create_or_update(new_topo) + topology_device = TopologyDeviceModel( + topology_uuid=endpoint_topology_uuid, + device_uuid=db_device.device_uuid) + self.database.create_or_update(topology_device) endpoint_attributes['topology_uuid'] = db_topology.topology_uuid new_endpoint = EndPointModel(**endpoint_attributes) - result : Tuple[EndPointModel, bool] = self.database.create_or_update(new_endpoint) + result: Tuple[EndPointModel, bool] = self.database.create_or_update(new_endpoint) db_endpoint, updated = result self.set_kpi_sample_types(db_endpoint, endpoint.kpi_sample_types) @@ -465,10 +498,15 @@ class ContextServiceServicerImpl(ContextServiceServicer): device_uuid = request.device_uuid.uuid with self.session() as session: - result = session.query(DeviceModel).filter_by(device_uuid=device_uuid).one_or_none() - if not result: + db_device = session.query(DeviceModel).filter_by(device_uuid=device_uuid).one_or_none() + + session.query(TopologyDeviceModel).filter_by(device_uuid=device_uuid).delete() + session.query(ConfigRuleModel).filter_by(config_uuid=db_device.device_config_uuid).delete() + session.query(ConfigModel).filter_by(config_uuid=db_device.device_config_uuid).delete() + + if not db_device: return Empty() - dict_device_id = result.dump_id() + dict_device_id = db_device.dump_id() session.query(DeviceModel).filter_by(device_uuid=device_uuid).delete() session.commit() @@ -496,19 +534,41 @@ class ContextServiceServicerImpl(ContextServiceServicer): @safe_and_metered_rpc_method(METRICS, LOGGER) def ListLinks(self, request: Empty, context : grpc.ServicerContext) -> LinkList: with self.session() as session: - result = session.query(DeviceModel).all() - return LinkList(links=[db_link.dump() for db_link in result]) + link_list = LinkList() + + db_links = session.query(LinkModel).all() + + for db_link in db_links: + link_uuid = db_link.link_uuid + filt = {'link_uuid': link_uuid} + link_endpoints = session.query(LinkEndPointModel).filter_by(**filt).all() + if link_endpoints: + eps = [] + for lep in link_endpoints: + filt = {'endpoint_uuid': lep.endpoint_uuid} + eps.append(session.query(EndPointModel).filter_by(**filt).one()) + link_list.links.append(Link(**db_link.dump(eps))) + + return link_list @safe_and_metered_rpc_method(METRICS, LOGGER) def GetLink(self, request: LinkId, context : grpc.ServicerContext) -> Link: link_uuid = request.link_uuid.uuid with self.session() as session: - result = session.query(LinkModel).filter(LinkModel.device_uuid == link_uuid).one_or_none() + result = session.query(LinkModel).filter(LinkModel.link_uuid == link_uuid).one_or_none() if not result: - raise NotFoundException(DeviceModel.__name__.replace('Model', ''), link_uuid) + raise NotFoundException(LinkModel.__name__.replace('Model', ''), link_uuid) - rd = result.dump() + filt = {'link_uuid': link_uuid} + link_endpoints = session.query(LinkEndPointModel).filter_by(**filt).all() + if link_endpoints: + eps = [] + for lep in link_endpoints: + filt = {'endpoint_uuid': lep.endpoint_uuid} + eps.append(session.query(EndPointModel).filter_by(**filt).one()) + return Link(**result.dump(eps)) + rd = result.dump() rt = Link(**rd) return rt @@ -520,7 +580,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): link_uuid = request.link_id.link_uuid.uuid new_link = LinkModel(**{ - 'lin_uuid': link_uuid + 'link_uuid': link_uuid }) result: Tuple[LinkModel, bool] = self.database.create_or_update(new_link) db_link, updated = result @@ -531,33 +591,20 @@ class ContextServiceServicerImpl(ContextServiceServicer): endpoint_topology_uuid = endpoint_id.topology_id.topology_uuid.uuid endpoint_topology_context_uuid = endpoint_id.topology_id.context_id.context_uuid.uuid - str_endpoint_key = key_to_str([endpoint_device_uuid, endpoint_uuid]) + db_topology = None if len(endpoint_topology_context_uuid) > 0 and len(endpoint_topology_uuid) > 0: - str_topology_key = key_to_str([endpoint_topology_context_uuid, endpoint_topology_uuid]) - # db_topology : TopologyModel = get_object(self.database, TopologyModel, str_topology_key) - db_topology : TopologyModel = self.database.get_object(TopologyModel, str_topology_key) - str_topology_device_key = key_to_str([str_topology_key, endpoint_device_uuid], separator='--') + db_topology: TopologyModel = self.database.get_object(TopologyModel, endpoint_topology_uuid) # check device is in topology - # get_object(self.database, TopologyDeviceModel, str_topology_device_key) - # str_endpoint_key = key_to_str([str_endpoint_key, str_topology_key], separator=':') - - # db_endpoint : EndPointModel = get_object(self.database, EndPointModel, str_endpoint_key) - LOGGER.info('str_endpoint_key: {}'.format(str_endpoint_key)) - db_endpoint: EndPointModel = self.database.get_object(EndPointModel, str_endpoint_key) - - # str_link_endpoint_key = key_to_str([link_uuid, endpoint_device_uuid], separator='--') - # result : Tuple[LinkEndPointModel, bool] = get_or_create_object( - # self.database, LinkEndPointModel, str_link_endpoint_key, { - # 'link_fk': db_link, 'endpoint_fk': db_endpoint}) - #db_link_endpoint, link_endpoint_created = result - - # if db_topology is not None: - # str_topology_link_key = key_to_str([str_topology_key, link_uuid], separator='--') - # result : Tuple[TopologyLinkModel, bool] = get_or_create_object( - # self.database, TopologyLinkModel, str_topology_link_key, { - # 'topology_fk': db_topology, 'link_fk': db_link}) - # #db_topology_link, topology_link_created = result + self.database.get_object(TopologyDeviceModel, endpoint_device_uuid) + + + link_endpoint = LinkEndPointModel(link_uuid=link_uuid, endpoint_uuid=endpoint_uuid) + result: Tuple[LinkEndPointModel, bool] = self.database.create_or_update(link_endpoint) + + if db_topology is not None: + topology_link = TopologyLinkModel(topology_uuid=endpoint_topology_uuid, link_uuid=link_uuid) + result: Tuple[TopologyLinkModel, bool] = self.database.create_or_update(topology_link) event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE dict_link_id = db_link.dump_id() @@ -566,15 +613,19 @@ class ContextServiceServicerImpl(ContextServiceServicer): @safe_and_metered_rpc_method(METRICS, LOGGER) def RemoveLink(self, request: LinkId, context : grpc.ServicerContext) -> Empty: - with self.lock: + with self.session() as session: link_uuid = request.link_uuid.uuid - db_link = LinkModel(self.database, link_uuid, auto_load=False) - found = db_link.load() - if not found: return Empty() - dict_link_id = db_link.dump_id() - db_link.delete() + session.query(TopologyLinkModel).filter_by(link_uuid=link_uuid).delete() + session.query(LinkEndPointModel).filter_by(link_uuid=link_uuid).delete() + + result = session.query(LinkModel).filter_by(link_uuid=link_uuid).one_or_none() + if not result: + return Empty() + dict_link_id = result.dump_id() + session.query(LinkModel).filter_by(link_uuid=link_uuid).delete() + session.commit() event_type = EventTypeEnum.EVENTTYPE_REMOVE notify_event(self.messagebroker, TOPIC_LINK, event_type, {'link_id': dict_link_id}) return Empty() @@ -584,7 +635,6 @@ class ContextServiceServicerImpl(ContextServiceServicer): for message in self.messagebroker.consume({TOPIC_LINK}, consume_timeout=CONSUME_TIMEOUT): yield LinkEvent(**json.loads(message.content)) - """ # ----- Service ---------------------------------------------------------------------------------------------------- @@ -693,6 +743,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): for message in self.messagebroker.consume({TOPIC_SERVICE}, consume_timeout=CONSUME_TIMEOUT): yield ServiceEvent(**json.loads(message.content)) + """ # ----- Slice ---------------------------------------------------------------------------------------------------- diff --git a/src/context/tests/Objects.py b/src/context/tests/Objects.py index 772da38e0..a2aebdd96 100644 --- a/src/context/tests/Objects.py +++ b/src/context/tests/Objects.py @@ -45,6 +45,7 @@ PACKET_PORT_SAMPLE_TYPES = [ # ----- Device --------------------------------------------------------------------------------------------------------- +EP1 = '5610e2c0-8abe-4127-80d0-7c68aff1c19e' EP2 = '7eb80584-2587-4e71-b10c-f3a5c48e84ab' EP3 = '368baf47-0540-4ab4-add8-a19b5167162c' EP100 = '6a923121-36e1-4b5e-8cd6-90aceca9b5cf' @@ -66,12 +67,12 @@ DEVICE_R1 = json_device_packetrouter_disabled( DEVICE_R1_UUID, endpoints=DEVICE_R1_EPS, config_rules=DEVICE_R1_RULES) -DEVICE_R2_UUID = 'R2' +DEVICE_R2_UUID = '2fd2be23-5b20-414c-b1ea-2f16ae6eb425' DEVICE_R2_ID = json_device_id(DEVICE_R2_UUID) DEVICE_R2_EPS = [ - json_endpoint(DEVICE_R2_ID, 'EP1', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), - json_endpoint(DEVICE_R2_ID, 'EP3', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), - json_endpoint(DEVICE_R2_ID, 'EP100', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R2_ID, EP1, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R2_ID, EP3, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R2_ID, EP100, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), ] DEVICE_R2_RULES = [ json_config_rule_set('dev/rsrc1/value', 'value4'), @@ -82,12 +83,12 @@ DEVICE_R2 = json_device_packetrouter_disabled( DEVICE_R2_UUID, endpoints=DEVICE_R2_EPS, config_rules=DEVICE_R2_RULES) -DEVICE_R3_UUID = 'R3' +DEVICE_R3_UUID = '3e71a251-2218-42c5-b4b8-de7760c0d9b3' DEVICE_R3_ID = json_device_id(DEVICE_R3_UUID) DEVICE_R3_EPS = [ - json_endpoint(DEVICE_R3_ID, 'EP1', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), - json_endpoint(DEVICE_R3_ID, 'EP2', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), - json_endpoint(DEVICE_R3_ID, 'EP100', '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R3_ID, EP2, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R3_ID, EP3, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), + json_endpoint(DEVICE_R3_ID, EP100, '10G', topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES), ] DEVICE_R3_RULES = [ json_config_rule_set('dev/rsrc1/value', 'value4'), @@ -99,29 +100,29 @@ DEVICE_R3 = json_device_packetrouter_disabled( # ----- Link ----------------------------------------------------------------------------------------------------------- -LINK_R1_R2_UUID = 'R1/EP2-R2/EP1' +LINK_R1_R2_UUID = 'c8f92eec-340e-4d31-8d7e-7074927dc889' LINK_R1_R2_ID = json_link_id(LINK_R1_R2_UUID) LINK_R1_R2_EPIDS = [ - json_endpoint_id(DEVICE_R1_ID, 'EP2', topology_id=TOPOLOGY_ID), - json_endpoint_id(DEVICE_R2_ID, 'EP1', topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R1_ID, EP2, topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R2_ID, EP1, topology_id=TOPOLOGY_ID), ] LINK_R1_R2 = json_link(LINK_R1_R2_UUID, LINK_R1_R2_EPIDS) -LINK_R2_R3_UUID = 'R2/EP3-R3/EP2' +LINK_R2_R3_UUID = 'f9e3539a-d8f9-4737-b4b4-cacf7f90aa5d' LINK_R2_R3_ID = json_link_id(LINK_R2_R3_UUID) LINK_R2_R3_EPIDS = [ - json_endpoint_id(DEVICE_R2_ID, 'EP3', topology_id=TOPOLOGY_ID), - json_endpoint_id(DEVICE_R3_ID, 'EP2', topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R2_ID, EP3, topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R3_ID, EP2, topology_id=TOPOLOGY_ID), ] LINK_R2_R3 = json_link(LINK_R2_R3_UUID, LINK_R2_R3_EPIDS) -LINK_R1_R3_UUID = 'R1/EP3-R3/EP1' +LINK_R1_R3_UUID = '1f1a988c-47a9-41b2-afd9-ebd6d434a0b4' LINK_R1_R3_ID = json_link_id(LINK_R1_R3_UUID) LINK_R1_R3_EPIDS = [ - json_endpoint_id(DEVICE_R1_ID, 'EP3', topology_id=TOPOLOGY_ID), - json_endpoint_id(DEVICE_R3_ID, 'EP1', topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R1_ID, EP3, topology_id=TOPOLOGY_ID), + json_endpoint_id(DEVICE_R3_ID, EP1, topology_id=TOPOLOGY_ID), ] LINK_R1_R3 = json_link(LINK_R1_R3_UUID, LINK_R1_R3_EPIDS) -- GitLab