Loading src/context/service/Database.py +3 −0 Original line number Diff line number Diff line Loading @@ -18,6 +18,9 @@ class Database(Session): return result def get_object(self): pass def clear(self): with self.session() as session: engine = session.get_bind() Loading src/context/service/database/ContextModel.py +4 −1 Original line number Diff line number Diff line Loading @@ -17,6 +17,7 @@ from typing import Dict, List from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID from context.service.database.Base import Base from sqlalchemy.orm import relationship LOGGER = logging.getLogger(__name__) Loading @@ -24,9 +25,11 @@ LOGGER = logging.getLogger(__name__) class ContextModel(Base): __tablename__ = 'Context' context_uuid = Column(UUID(as_uuid=False), primary_key=True) # Relationships topology = relationship("TopologyModel", back_populates="context") def dump_id(self) -> Dict: return {'context_uuid': {'uuid': self.context_uuid}} Loading src/context/service/database/TopologyModel.py +16 −10 Original line number Diff line number Diff line Loading @@ -19,23 +19,28 @@ from common.orm.fields.PrimaryKeyField import PrimaryKeyField from common.orm.fields.StringField import StringField from common.orm.model.Model import Model from common.orm.HighLevel import get_related_objects from .ContextModel import ContextModel from sqlalchemy.orm import relationship from sqlalchemy import Column, ForeignKey from sqlalchemy.dialects.postgresql import UUID from context.service.database.Base import Base LOGGER = logging.getLogger(__name__) class TopologyModel(Model): pk = PrimaryKeyField() context_fk = ForeignKeyField(ContextModel) topology_uuid = StringField(required=True, allow_empty=False) class TopologyModel(Base): __tablename__ = 'Topology' context_fk = Column(UUID(as_uuid=False), ForeignKey("Context.context_uuid"), nullable=False) topology_uuid = Column(UUID(as_uuid=False), primary_key=True, nullable=False) # Relationships context = relationship("ContextModel", back_populates="topology", lazy="joined") def dump_id(self) -> Dict: context_id = ContextModel(self.database, self.context_fk).dump_id() context_id = self.context.dump_id() return { 'context_id': context_id, 'topology_uuid': {'uuid': self.topology_uuid}, } def dump_device_ids(self) -> List[Dict]: """def dump_device_ids(self) -> List[Dict]: from .RelationModels import TopologyDeviceModel # pylint: disable=import-outside-toplevel db_devices = get_related_objects(self, TopologyDeviceModel, 'device_fk') return [db_device.dump_id() for db_device in sorted(db_devices, key=operator.attrgetter('pk'))] Loading @@ -44,11 +49,12 @@ class TopologyModel(Model): from .RelationModels import TopologyLinkModel # pylint: disable=import-outside-toplevel db_links = get_related_objects(self, TopologyLinkModel, 'link_fk') return [db_link.dump_id() for db_link in sorted(db_links, key=operator.attrgetter('pk'))] """ def dump( # pylint: disable=arguments-differ self, include_devices=True, include_links=True ) -> Dict: result = {'topology_id': self.dump_id()} if include_devices: result['device_ids'] = self.dump_device_ids() if include_links: result['link_ids'] = self.dump_link_ids() # if include_devices: result['device_ids'] = self.dump_device_ids() # if include_links: result['link_ids'] = self.dump_link_ids() return result src/context/service/grpc_server/ContextServiceServicerImpl.py +66 −64 Original line number Diff line number Diff line Loading @@ -15,10 +15,8 @@ import grpc, json, logging, operator, threading from typing import Iterator, List, Set, Tuple from common.message_broker.MessageBroker import MessageBroker from common.orm.Database import Database from common.orm.HighLevel import ( get_all_objects, get_object, get_or_create_object, get_related_objects, update_or_create_object) from common.orm.backend.Tools import key_to_str from context.service.Database import Database from common.proto.context_pb2 import ( Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList, Context, ContextEvent, ContextId, ContextIdList, ContextList, Loading @@ -31,9 +29,10 @@ from common.proto.context_pb2 import ( from common.proto.context_pb2_grpc import ContextServiceServicer from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException from sqlalchemy.orm import Session from sqlalchemy.orm import Session, contains_eager, selectinload from common.rpc_method_wrapper.ServiceExceptions import NotFoundException """ from context.service.database.ConfigModel import grpc_config_rules_to_raw, update_config from context.service.database.ConnectionModel import ConnectionModel, set_path Loading @@ -51,6 +50,7 @@ from context.service.database.SliceModel import SliceModel, grpc_to_enum__slice_ from context.service.database.TopologyModel import TopologyModel """ from context.service.database.ContextModel import ContextModel from context.service.database.TopologyModel import TopologyModel # from context.service.database.TopologyModel import TopologyModel from context.service.database.Events import notify_event Loading @@ -77,6 +77,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): LOGGER.debug('Creating Servicer...') self.lock = threading.Lock() self.session = session self.database = Database(session) self.messagebroker = messagebroker LOGGER.debug('Servicer Created') Loading Loading @@ -135,8 +136,6 @@ class ContextServiceServicerImpl(ContextServiceServicer): result = session.query(ContextModel).filter_by(context_uuid=context_uuid).all() if not result: updated = False with self.session() as session: session.merge(context_add) session.commit() Loading @@ -161,7 +160,6 @@ class ContextServiceServicerImpl(ContextServiceServicer): notify_event(self.messagebroker, TOPIC_CONTEXT, event_type, {'context_id': result.dump_id()}) return Empty() """ @safe_and_metered_rpc_method(METRICS, LOGGER) def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]: for message in self.messagebroker.consume({TOPIC_CONTEXT}, consume_timeout=CONSUME_TIMEOUT): Loading @@ -174,75 +172,78 @@ class ContextServiceServicerImpl(ContextServiceServicer): def ListTopologyIds(self, request: ContextId, context : grpc.ServicerContext) -> TopologyIdList: with self.lock: context_uuid = request.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) db_topologies : Set[TopologyModel] = get_related_objects(db_context, TopologyModel) db_topologies = sorted(db_topologies, key=operator.attrgetter('pk')) with self.session() as session: result = session.query(ContextModel).options(selectinload(ContextModel.topology)).filter_by(context_uuid=context_uuid).one_or_none() if not result: raise NotFoundException(ContextModel.__name__.replace('Model', ''), context_uuid) db_topologies = result.topology return TopologyIdList(topology_ids=[db_topology.dump_id() for db_topology in db_topologies]) @safe_and_metered_rpc_method(METRICS, LOGGER) def ListTopologies(self, request: ContextId, context : grpc.ServicerContext) -> TopologyList: with self.lock: context_uuid = request.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) db_topologies : Set[TopologyModel] = get_related_objects(db_context, TopologyModel) db_topologies = sorted(db_topologies, key=operator.attrgetter('pk')) with self.session() as session: result = session.query(ContextModel).options(selectinload(ContextModel.topology)).filter_by( context_uuid=context_uuid).one_or_none() if not result: raise NotFoundException(ContextModel.__name__.replace('Model', ''), context_uuid) db_topologies = result.topology return TopologyList(topologies=[db_topology.dump() for db_topology in db_topologies]) @safe_and_metered_rpc_method(METRICS, LOGGER) def GetTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Topology: with self.lock: str_key = key_to_str([request.context_id.context_uuid.uuid, request.topology_uuid.uuid]) db_topology : TopologyModel = get_object(self.database, TopologyModel, str_key) return Topology(**db_topology.dump(include_devices=True, include_links=True)) def GetTopology(self, request: TopologyId, contextt : grpc.ServicerContext) -> Topology: context_uuid = request.context_id.context_uuid.uuid topology_uuid = request.topology_uuid.uuid with self.session() as session: result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).options(contains_eager(TopologyModel.context)).one_or_none() if not result: raise NotFoundException(TopologyModel.__name__.replace('Model', ''), topology_uuid) return Topology(**result.dump()) @safe_and_metered_rpc_method(METRICS, LOGGER) def SetTopology(self, request: Topology, context : grpc.ServicerContext) -> TopologyId: with self.lock: context_uuid = request.topology_id.context_id.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) topology_uuid = request.topology_id.topology_uuid.uuid str_topology_key = key_to_str([context_uuid, topology_uuid]) result : Tuple[TopologyModel, bool] = update_or_create_object( self.database, TopologyModel, str_topology_key, { 'context_fk': db_context, 'topology_uuid': topology_uuid}) db_topology,updated = result for device_id in request.device_ids: device_uuid = device_id.device_uuid.uuid db_device = get_object(self.database, DeviceModel, device_uuid) str_topology_device_key = key_to_str([str_topology_key, device_uuid], separator='--') result : Tuple[TopologyDeviceModel, bool] = update_or_create_object( self.database, TopologyDeviceModel, str_topology_device_key, {'topology_fk': db_topology, 'device_fk': db_device}) #db_topology_device,topology_device_updated = result with self.session() as session: db_context: ContextModel = session.query(ContextModel).filter_by(context_uuid=context_uuid).one() for link_id in request.link_ids: link_uuid = link_id.link_uuid.uuid db_link = get_object(self.database, LinkModel, link_uuid) topology_add = TopologyModel(topology_uuid=topology_uuid, context_fk=context_uuid) topology_add.context = db_context updated = True with self.session() as session: result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).options(contains_eager(TopologyModel.context)).one_or_none() str_topology_link_key = key_to_str([str_topology_key, link_uuid], separator='--') result : Tuple[TopologyLinkModel, bool] = update_or_create_object( self.database, TopologyLinkModel, str_topology_link_key, {'topology_fk': db_topology, 'link_fk': db_link}) #db_topology_link,topology_link_updated = result if not result: updated = False session.merge(topology_add) session.commit() event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE dict_topology_id = db_topology.dump_id() dict_topology_id = topology_add.dump_id() notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': dict_topology_id}) return TopologyId(**dict_topology_id) @safe_and_metered_rpc_method(METRICS, LOGGER) def RemoveTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Empty: with self.lock: context_uuid = request.context_id.context_uuid.uuid topology_uuid = request.topology_uuid.uuid db_topology = TopologyModel(self.database, key_to_str([context_uuid, topology_uuid]), auto_load=False) found = db_topology.load() if not found: return Empty() dict_topology_id = db_topology.dump_id() db_topology.delete() with self.session() as session: result = session.query(TopologyModel).filter_by(topology_uuid=topology_uuid, context_fk=context_uuid).one_or_none() if not result: return Empty() dict_topology_id = result.dump_id() session.query(TopologyModel).filter_by(topology_uuid=topology_uuid, context_fk=context_uuid).delete() session.commit() event_type = EventTypeEnum.EVENTTYPE_REMOVE notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': dict_topology_id}) return Empty() Loading @@ -251,6 +252,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): def GetTopologyEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[TopologyEvent]: for message in self.messagebroker.consume({TOPIC_TOPOLOGY}, consume_timeout=CONSUME_TIMEOUT): yield TopologyEvent(**json.loads(message.content)) """ # ----- Device ----------------------------------------------------------------------------------------------------- Loading src/context/tests/test_unitary.py +47 −59 Original line number Diff line number Diff line Loading @@ -44,6 +44,7 @@ from requests import Session from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from context.service.database.ContextModel import ContextModel from context.service.database.TopologyModel import TopologyModel from context.service.database.Base import Base from .Objects import ( Loading Loading @@ -76,15 +77,6 @@ REDIS_CONFIG = { SCENARIOS = [ ('all_sqlalchemy', {}, MessageBrokerBackendEnum.INMEMORY, {} ), ] @pytest.fixture(scope='session', ids=[str(scenario[0]) for scenario in SCENARIOS], params=SCENARIOS) def context_db_mb(request) -> Tuple[Session, MessageBroker]: name,db_backend,db_settings,mb_backend,mb_settings = request.param msg = 'Running scenario {:s} db_backend={:s}, db_settings={:s}, mb_backend={:s}, mb_settings={:s}...' LOGGER.info(msg.format(str(name), str(db_backend.value), str(db_settings), str(mb_backend.value), str(mb_settings))) _database = Database(get_database_backend(backend=db_backend, **db_settings)) _message_broker = MessageBroker(get_messagebroker_backend(backend=mb_backend, **mb_settings)) yield _database, _message_broker _message_broker.terminate() @pytest.fixture(scope='session', ids=[str(scenario[0]) for scenario in SCENARIOS], params=SCENARIOS) def context_s_mb(request) -> Tuple[Session, MessageBroker]: Loading Loading @@ -207,23 +199,19 @@ def test_grpc_context( assert e.value.details() == msg # ----- Check create event ----------------------------------------------------------------------------------------- """ event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID """ # ----- Update the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetContext(Context(**CONTEXT)) assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Check update event ----------------------------------------------------------------------------------------- """ event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID """ # ----- Dump state of database after create/update the object ------------------------------------------------------ db_entries = database.query_all(ContextModel) Loading Loading @@ -271,15 +259,16 @@ def test_grpc_context( # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 0 """ def test_grpc_topology( context_client_grpc: ContextClient, # pylint: disable=redefined-outer-name context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name context_database = context_db_mb[0] context_s_mb: Tuple[Session, MessageBroker]): # pylint: disable=redefined-outer-name session = context_s_mb[0] database = Database(session) # ----- Clean the database ----------------------------------------------------------------------------------------- context_database.clear_all() database.clear() # ----- Initialize the EventsCollector ----------------------------------------------------------------------------- events_collector = EventsCollector(context_client_grpc) Loading @@ -288,32 +277,30 @@ def test_grpc_topology( # ----- Prepare dependencies for the test and capture related events ----------------------------------------------- response = context_client_grpc.SetContext(Context(**CONTEXT)) assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # event = events_collector.get_event(block=True) # assert isinstance(event, ContextEvent) # assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE # assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Get when the object does not exist ------------------------------------------------------------------------- with pytest.raises(grpc.RpcError) as e: context_client_grpc.GetTopology(TopologyId(**TOPOLOGY_ID)) assert e.value.code() == grpc.StatusCode.NOT_FOUND assert e.value.details() == 'Topology({:s}/{:s}) not found'.format(DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID) # assert e.value.details() == 'Topology({:s}/{:s}) not found'.format(DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID) assert e.value.details() == 'Topology({:s}) not found'.format(DEFAULT_TOPOLOGY_UUID) # ----- List when the object does not exist ------------------------------------------------------------------------ response = context_client_grpc.ListTopologyIds(ContextId(**CONTEXT_ID)) assert len(response.topology_ids) == 0 response = context_client_grpc.ListTopologies(ContextId(**CONTEXT_ID)) assert len(response.topologies) == 0 # ----- Dump state of database before create the object ------------------------------------------------------------ db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 2 assert len(db_entries) == 0 # ----- Create the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetTopology(Topology(**TOPOLOGY)) Loading @@ -326,16 +313,16 @@ def test_grpc_topology( assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Check create event ----------------------------------------------------------------------------------------- events = events_collector.get_events(block=True, count=2) # events = events_collector.get_events(block=True, count=2) assert isinstance(events[0], TopologyEvent) assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # assert isinstance(events[0], TopologyEvent) # assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_CREATE # assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID assert isinstance(events[1], ContextEvent) assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert isinstance(events[1], ContextEvent) # assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_UPDATE # assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Update the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetTopology(Topology(**TOPOLOGY)) Loading @@ -343,19 +330,19 @@ def test_grpc_topology( assert response.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # ----- Check update event ----------------------------------------------------------------------------------------- event = events_collector.get_event(block=True) assert isinstance(event, TopologyEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert event.topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert event.topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # event = events_collector.get_event(block=True) # assert isinstance(event, TopologyEvent) # assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE # assert event.topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert event.topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # ----- Dump state of database after create/update the object ------------------------------------------------------ db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 5 assert len(db_entries) == 1 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetTopology(TopologyId(**TOPOLOGY_ID)) Loading @@ -382,28 +369,29 @@ def test_grpc_topology( context_client_grpc.RemoveContext(ContextId(**CONTEXT_ID)) # ----- Check remove event ----------------------------------------------------------------------------------------- events = events_collector.get_events(block=True, count=2) # events = events_collector.get_events(block=True, count=2) assert isinstance(events[0], TopologyEvent) assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # assert isinstance(events[0], TopologyEvent) # assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE # assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID assert isinstance(events[1], ContextEvent) assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert isinstance(events[1], ContextEvent) # assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE # assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Stop the EventsCollector ----------------------------------------------------------------------------------- events_collector.stop() # events_collector.stop() # ----- Dump state of database after remove the object ------------------------------------------------------------- db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 0 """ def test_grpc_device( context_client_grpc : ContextClient, # pylint: disable=redefined-outer-name Loading Loading
src/context/service/Database.py +3 −0 Original line number Diff line number Diff line Loading @@ -18,6 +18,9 @@ class Database(Session): return result def get_object(self): pass def clear(self): with self.session() as session: engine = session.get_bind() Loading
src/context/service/database/ContextModel.py +4 −1 Original line number Diff line number Diff line Loading @@ -17,6 +17,7 @@ from typing import Dict, List from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID from context.service.database.Base import Base from sqlalchemy.orm import relationship LOGGER = logging.getLogger(__name__) Loading @@ -24,9 +25,11 @@ LOGGER = logging.getLogger(__name__) class ContextModel(Base): __tablename__ = 'Context' context_uuid = Column(UUID(as_uuid=False), primary_key=True) # Relationships topology = relationship("TopologyModel", back_populates="context") def dump_id(self) -> Dict: return {'context_uuid': {'uuid': self.context_uuid}} Loading
src/context/service/database/TopologyModel.py +16 −10 Original line number Diff line number Diff line Loading @@ -19,23 +19,28 @@ from common.orm.fields.PrimaryKeyField import PrimaryKeyField from common.orm.fields.StringField import StringField from common.orm.model.Model import Model from common.orm.HighLevel import get_related_objects from .ContextModel import ContextModel from sqlalchemy.orm import relationship from sqlalchemy import Column, ForeignKey from sqlalchemy.dialects.postgresql import UUID from context.service.database.Base import Base LOGGER = logging.getLogger(__name__) class TopologyModel(Model): pk = PrimaryKeyField() context_fk = ForeignKeyField(ContextModel) topology_uuid = StringField(required=True, allow_empty=False) class TopologyModel(Base): __tablename__ = 'Topology' context_fk = Column(UUID(as_uuid=False), ForeignKey("Context.context_uuid"), nullable=False) topology_uuid = Column(UUID(as_uuid=False), primary_key=True, nullable=False) # Relationships context = relationship("ContextModel", back_populates="topology", lazy="joined") def dump_id(self) -> Dict: context_id = ContextModel(self.database, self.context_fk).dump_id() context_id = self.context.dump_id() return { 'context_id': context_id, 'topology_uuid': {'uuid': self.topology_uuid}, } def dump_device_ids(self) -> List[Dict]: """def dump_device_ids(self) -> List[Dict]: from .RelationModels import TopologyDeviceModel # pylint: disable=import-outside-toplevel db_devices = get_related_objects(self, TopologyDeviceModel, 'device_fk') return [db_device.dump_id() for db_device in sorted(db_devices, key=operator.attrgetter('pk'))] Loading @@ -44,11 +49,12 @@ class TopologyModel(Model): from .RelationModels import TopologyLinkModel # pylint: disable=import-outside-toplevel db_links = get_related_objects(self, TopologyLinkModel, 'link_fk') return [db_link.dump_id() for db_link in sorted(db_links, key=operator.attrgetter('pk'))] """ def dump( # pylint: disable=arguments-differ self, include_devices=True, include_links=True ) -> Dict: result = {'topology_id': self.dump_id()} if include_devices: result['device_ids'] = self.dump_device_ids() if include_links: result['link_ids'] = self.dump_link_ids() # if include_devices: result['device_ids'] = self.dump_device_ids() # if include_links: result['link_ids'] = self.dump_link_ids() return result
src/context/service/grpc_server/ContextServiceServicerImpl.py +66 −64 Original line number Diff line number Diff line Loading @@ -15,10 +15,8 @@ import grpc, json, logging, operator, threading from typing import Iterator, List, Set, Tuple from common.message_broker.MessageBroker import MessageBroker from common.orm.Database import Database from common.orm.HighLevel import ( get_all_objects, get_object, get_or_create_object, get_related_objects, update_or_create_object) from common.orm.backend.Tools import key_to_str from context.service.Database import Database from common.proto.context_pb2 import ( Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList, Context, ContextEvent, ContextId, ContextIdList, ContextList, Loading @@ -31,9 +29,10 @@ from common.proto.context_pb2 import ( from common.proto.context_pb2_grpc import ContextServiceServicer from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException from sqlalchemy.orm import Session from sqlalchemy.orm import Session, contains_eager, selectinload from common.rpc_method_wrapper.ServiceExceptions import NotFoundException """ from context.service.database.ConfigModel import grpc_config_rules_to_raw, update_config from context.service.database.ConnectionModel import ConnectionModel, set_path Loading @@ -51,6 +50,7 @@ from context.service.database.SliceModel import SliceModel, grpc_to_enum__slice_ from context.service.database.TopologyModel import TopologyModel """ from context.service.database.ContextModel import ContextModel from context.service.database.TopologyModel import TopologyModel # from context.service.database.TopologyModel import TopologyModel from context.service.database.Events import notify_event Loading @@ -77,6 +77,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): LOGGER.debug('Creating Servicer...') self.lock = threading.Lock() self.session = session self.database = Database(session) self.messagebroker = messagebroker LOGGER.debug('Servicer Created') Loading Loading @@ -135,8 +136,6 @@ class ContextServiceServicerImpl(ContextServiceServicer): result = session.query(ContextModel).filter_by(context_uuid=context_uuid).all() if not result: updated = False with self.session() as session: session.merge(context_add) session.commit() Loading @@ -161,7 +160,6 @@ class ContextServiceServicerImpl(ContextServiceServicer): notify_event(self.messagebroker, TOPIC_CONTEXT, event_type, {'context_id': result.dump_id()}) return Empty() """ @safe_and_metered_rpc_method(METRICS, LOGGER) def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]: for message in self.messagebroker.consume({TOPIC_CONTEXT}, consume_timeout=CONSUME_TIMEOUT): Loading @@ -174,75 +172,78 @@ class ContextServiceServicerImpl(ContextServiceServicer): def ListTopologyIds(self, request: ContextId, context : grpc.ServicerContext) -> TopologyIdList: with self.lock: context_uuid = request.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) db_topologies : Set[TopologyModel] = get_related_objects(db_context, TopologyModel) db_topologies = sorted(db_topologies, key=operator.attrgetter('pk')) with self.session() as session: result = session.query(ContextModel).options(selectinload(ContextModel.topology)).filter_by(context_uuid=context_uuid).one_or_none() if not result: raise NotFoundException(ContextModel.__name__.replace('Model', ''), context_uuid) db_topologies = result.topology return TopologyIdList(topology_ids=[db_topology.dump_id() for db_topology in db_topologies]) @safe_and_metered_rpc_method(METRICS, LOGGER) def ListTopologies(self, request: ContextId, context : grpc.ServicerContext) -> TopologyList: with self.lock: context_uuid = request.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) db_topologies : Set[TopologyModel] = get_related_objects(db_context, TopologyModel) db_topologies = sorted(db_topologies, key=operator.attrgetter('pk')) with self.session() as session: result = session.query(ContextModel).options(selectinload(ContextModel.topology)).filter_by( context_uuid=context_uuid).one_or_none() if not result: raise NotFoundException(ContextModel.__name__.replace('Model', ''), context_uuid) db_topologies = result.topology return TopologyList(topologies=[db_topology.dump() for db_topology in db_topologies]) @safe_and_metered_rpc_method(METRICS, LOGGER) def GetTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Topology: with self.lock: str_key = key_to_str([request.context_id.context_uuid.uuid, request.topology_uuid.uuid]) db_topology : TopologyModel = get_object(self.database, TopologyModel, str_key) return Topology(**db_topology.dump(include_devices=True, include_links=True)) def GetTopology(self, request: TopologyId, contextt : grpc.ServicerContext) -> Topology: context_uuid = request.context_id.context_uuid.uuid topology_uuid = request.topology_uuid.uuid with self.session() as session: result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).options(contains_eager(TopologyModel.context)).one_or_none() if not result: raise NotFoundException(TopologyModel.__name__.replace('Model', ''), topology_uuid) return Topology(**result.dump()) @safe_and_metered_rpc_method(METRICS, LOGGER) def SetTopology(self, request: Topology, context : grpc.ServicerContext) -> TopologyId: with self.lock: context_uuid = request.topology_id.context_id.context_uuid.uuid db_context : ContextModel = get_object(self.database, ContextModel, context_uuid) topology_uuid = request.topology_id.topology_uuid.uuid str_topology_key = key_to_str([context_uuid, topology_uuid]) result : Tuple[TopologyModel, bool] = update_or_create_object( self.database, TopologyModel, str_topology_key, { 'context_fk': db_context, 'topology_uuid': topology_uuid}) db_topology,updated = result for device_id in request.device_ids: device_uuid = device_id.device_uuid.uuid db_device = get_object(self.database, DeviceModel, device_uuid) str_topology_device_key = key_to_str([str_topology_key, device_uuid], separator='--') result : Tuple[TopologyDeviceModel, bool] = update_or_create_object( self.database, TopologyDeviceModel, str_topology_device_key, {'topology_fk': db_topology, 'device_fk': db_device}) #db_topology_device,topology_device_updated = result with self.session() as session: db_context: ContextModel = session.query(ContextModel).filter_by(context_uuid=context_uuid).one() for link_id in request.link_ids: link_uuid = link_id.link_uuid.uuid db_link = get_object(self.database, LinkModel, link_uuid) topology_add = TopologyModel(topology_uuid=topology_uuid, context_fk=context_uuid) topology_add.context = db_context updated = True with self.session() as session: result = session.query(TopologyModel).join(TopologyModel.context).filter(TopologyModel.topology_uuid==topology_uuid).options(contains_eager(TopologyModel.context)).one_or_none() str_topology_link_key = key_to_str([str_topology_key, link_uuid], separator='--') result : Tuple[TopologyLinkModel, bool] = update_or_create_object( self.database, TopologyLinkModel, str_topology_link_key, {'topology_fk': db_topology, 'link_fk': db_link}) #db_topology_link,topology_link_updated = result if not result: updated = False session.merge(topology_add) session.commit() event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE dict_topology_id = db_topology.dump_id() dict_topology_id = topology_add.dump_id() notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': dict_topology_id}) return TopologyId(**dict_topology_id) @safe_and_metered_rpc_method(METRICS, LOGGER) def RemoveTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Empty: with self.lock: context_uuid = request.context_id.context_uuid.uuid topology_uuid = request.topology_uuid.uuid db_topology = TopologyModel(self.database, key_to_str([context_uuid, topology_uuid]), auto_load=False) found = db_topology.load() if not found: return Empty() dict_topology_id = db_topology.dump_id() db_topology.delete() with self.session() as session: result = session.query(TopologyModel).filter_by(topology_uuid=topology_uuid, context_fk=context_uuid).one_or_none() if not result: return Empty() dict_topology_id = result.dump_id() session.query(TopologyModel).filter_by(topology_uuid=topology_uuid, context_fk=context_uuid).delete() session.commit() event_type = EventTypeEnum.EVENTTYPE_REMOVE notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': dict_topology_id}) return Empty() Loading @@ -251,6 +252,7 @@ class ContextServiceServicerImpl(ContextServiceServicer): def GetTopologyEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[TopologyEvent]: for message in self.messagebroker.consume({TOPIC_TOPOLOGY}, consume_timeout=CONSUME_TIMEOUT): yield TopologyEvent(**json.loads(message.content)) """ # ----- Device ----------------------------------------------------------------------------------------------------- Loading
src/context/tests/test_unitary.py +47 −59 Original line number Diff line number Diff line Loading @@ -44,6 +44,7 @@ from requests import Session from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from context.service.database.ContextModel import ContextModel from context.service.database.TopologyModel import TopologyModel from context.service.database.Base import Base from .Objects import ( Loading Loading @@ -76,15 +77,6 @@ REDIS_CONFIG = { SCENARIOS = [ ('all_sqlalchemy', {}, MessageBrokerBackendEnum.INMEMORY, {} ), ] @pytest.fixture(scope='session', ids=[str(scenario[0]) for scenario in SCENARIOS], params=SCENARIOS) def context_db_mb(request) -> Tuple[Session, MessageBroker]: name,db_backend,db_settings,mb_backend,mb_settings = request.param msg = 'Running scenario {:s} db_backend={:s}, db_settings={:s}, mb_backend={:s}, mb_settings={:s}...' LOGGER.info(msg.format(str(name), str(db_backend.value), str(db_settings), str(mb_backend.value), str(mb_settings))) _database = Database(get_database_backend(backend=db_backend, **db_settings)) _message_broker = MessageBroker(get_messagebroker_backend(backend=mb_backend, **mb_settings)) yield _database, _message_broker _message_broker.terminate() @pytest.fixture(scope='session', ids=[str(scenario[0]) for scenario in SCENARIOS], params=SCENARIOS) def context_s_mb(request) -> Tuple[Session, MessageBroker]: Loading Loading @@ -207,23 +199,19 @@ def test_grpc_context( assert e.value.details() == msg # ----- Check create event ----------------------------------------------------------------------------------------- """ event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID """ # ----- Update the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetContext(Context(**CONTEXT)) assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Check update event ----------------------------------------------------------------------------------------- """ event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID """ # ----- Dump state of database after create/update the object ------------------------------------------------------ db_entries = database.query_all(ContextModel) Loading Loading @@ -271,15 +259,16 @@ def test_grpc_context( # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 0 """ def test_grpc_topology( context_client_grpc: ContextClient, # pylint: disable=redefined-outer-name context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name context_database = context_db_mb[0] context_s_mb: Tuple[Session, MessageBroker]): # pylint: disable=redefined-outer-name session = context_s_mb[0] database = Database(session) # ----- Clean the database ----------------------------------------------------------------------------------------- context_database.clear_all() database.clear() # ----- Initialize the EventsCollector ----------------------------------------------------------------------------- events_collector = EventsCollector(context_client_grpc) Loading @@ -288,32 +277,30 @@ def test_grpc_topology( # ----- Prepare dependencies for the test and capture related events ----------------------------------------------- response = context_client_grpc.SetContext(Context(**CONTEXT)) assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID event = events_collector.get_event(block=True) assert isinstance(event, ContextEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # event = events_collector.get_event(block=True) # assert isinstance(event, ContextEvent) # assert event.event.event_type == EventTypeEnum.EVENTTYPE_CREATE # assert event.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Get when the object does not exist ------------------------------------------------------------------------- with pytest.raises(grpc.RpcError) as e: context_client_grpc.GetTopology(TopologyId(**TOPOLOGY_ID)) assert e.value.code() == grpc.StatusCode.NOT_FOUND assert e.value.details() == 'Topology({:s}/{:s}) not found'.format(DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID) # assert e.value.details() == 'Topology({:s}/{:s}) not found'.format(DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID) assert e.value.details() == 'Topology({:s}) not found'.format(DEFAULT_TOPOLOGY_UUID) # ----- List when the object does not exist ------------------------------------------------------------------------ response = context_client_grpc.ListTopologyIds(ContextId(**CONTEXT_ID)) assert len(response.topology_ids) == 0 response = context_client_grpc.ListTopologies(ContextId(**CONTEXT_ID)) assert len(response.topologies) == 0 # ----- Dump state of database before create the object ------------------------------------------------------------ db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 2 assert len(db_entries) == 0 # ----- Create the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetTopology(Topology(**TOPOLOGY)) Loading @@ -326,16 +313,16 @@ def test_grpc_topology( assert response.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Check create event ----------------------------------------------------------------------------------------- events = events_collector.get_events(block=True, count=2) # events = events_collector.get_events(block=True, count=2) assert isinstance(events[0], TopologyEvent) assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_CREATE assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # assert isinstance(events[0], TopologyEvent) # assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_CREATE # assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID assert isinstance(events[1], ContextEvent) assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert isinstance(events[1], ContextEvent) # assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_UPDATE # assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Update the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetTopology(Topology(**TOPOLOGY)) Loading @@ -343,19 +330,19 @@ def test_grpc_topology( assert response.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # ----- Check update event ----------------------------------------------------------------------------------------- event = events_collector.get_event(block=True) assert isinstance(event, TopologyEvent) assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE assert event.topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert event.topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # event = events_collector.get_event(block=True) # assert isinstance(event, TopologyEvent) # assert event.event.event_type == EventTypeEnum.EVENTTYPE_UPDATE # assert event.topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert event.topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # ----- Dump state of database after create/update the object ------------------------------------------------------ db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 5 assert len(db_entries) == 1 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetTopology(TopologyId(**TOPOLOGY_ID)) Loading @@ -382,28 +369,29 @@ def test_grpc_topology( context_client_grpc.RemoveContext(ContextId(**CONTEXT_ID)) # ----- Check remove event ----------------------------------------------------------------------------------------- events = events_collector.get_events(block=True, count=2) # events = events_collector.get_events(block=True, count=2) assert isinstance(events[0], TopologyEvent) assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID # assert isinstance(events[0], TopologyEvent) # assert events[0].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE # assert events[0].topology_id.context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert events[0].topology_id.topology_uuid.uuid == DEFAULT_TOPOLOGY_UUID assert isinstance(events[1], ContextEvent) assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # assert isinstance(events[1], ContextEvent) # assert events[1].event.event_type == EventTypeEnum.EVENTTYPE_REMOVE # assert events[1].context_id.context_uuid.uuid == DEFAULT_CONTEXT_UUID # ----- Stop the EventsCollector ----------------------------------------------------------------------------------- events_collector.stop() # events_collector.stop() # ----- Dump state of database after remove the object ------------------------------------------------------------- db_entries = context_database.dump() db_entries = database.query_all(TopologyModel) LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries))) for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover # for db_entry in db_entries: # LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') assert len(db_entries) == 0 """ def test_grpc_device( context_client_grpc : ContextClient, # pylint: disable=redefined-outer-name Loading