Resolve "Notificatoins from Context are incomplete"

Closes #54 (closed)

Common - Message Broker - NATS:

  • Improved subscriber error handling and logging
  • Minor linting improvement

Common - Tests - MockMessageBroker:

  • Removed unused TOPICS variable

Common - Tools - Context Queries:

  • Updated method create_topology() to enable defining the topology name
  • Implemented get_topology_details() method

Context Component:

  • Moved generation of events to the database sub-module
  • Improved API for notification publication and consumption
  • Moved construction of gRPC reply objects to the database sub-module
  • Moved generation of change notifications to database sub-modules
  • Activated reversed relations in TopologyDeviceModel and TopologyLinkModel
  • Extended Topology to notify changes on Context when Topology is modified or deleted
  • Extended Device to notify changes on Topology and Context when Device is modified or deleted
  • Extended Link to notify changes on Topology and Context when Link is modified or deleted
  • Extended Link to report changes when Endpoints are added.
  • Extended Service to notify changes on Context when Service is modified or deleted
  • Extended Slice to notify changes on Context when Slice is modified or deleted
  • Minor cosmetic improvements
Edited by Lluis Gifre Renom

Merge request reports

Loading
+6 −2
Changes for src/common/message_broker/backend/nats/NatsBackendThread.py: 6 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -52,9 +52,11 @@ class NatsBackendThread(threading.Thread):
        self, topic_name : str, timeout : float, out_queue : queue.Queue[Message], unsubscribe : threading.Event,
        ready_event : threading.Event
    ) -> None:
        try:
            LOGGER.info('[_run_subscriber] NATS URI: {:s}'.format(str(self._nats_uri)))
            client = await nats.connect(servers=[self._nats_uri])
        LOGGER.info('[_run_subscriber] Connected!')
            server_version = client.connected_server_version
            LOGGER.info('[_run_subscriber] Connected! NATS Server version: {:s}'.format(str(repr(server_version))))
            subscription = await client.subscribe(topic_name)
            LOGGER.info('[_run_subscriber] Subscribed!')
            ready_event.set()
@@ -68,6 +70,8 @@ class NatsBackendThread(threading.Thread):
                out_queue.put(Message(message.subject, message.data.decode('UTF-8')))
            await subscription.unsubscribe()
            await client.drain()
        except Exception:   # pylint: disable=broad-exception-caught
            LOGGER.exception('[_run_subscriber] Unhandled Exception')

    def subscribe(
        self, topic_name : str, timeout : float, out_queue : queue.Queue[Message], unsubscribe : threading.Event
@@ -79,7 +83,7 @@ class NatsBackendThread(threading.Thread):
        self._tasks.append(task)
        LOGGER.info('[subscribe] Waiting for subscriber to be ready...')
        is_ready = ready_event.wait(timeout=120)
        LOGGER.info('[subscribe] Subscriber Ready: {:s}'.format(str(is_ready)))
        LOGGER.info('[subscribe] Subscriber is Ready? {:s}'.format(str(is_ready)))

    def run(self) -> None:
        asyncio.set_event_loop(self._event_loop)
+0 −4
Changes for src/common/tests/MockMessageBroker.py: 0 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -28,10 +28,6 @@ TOPIC_SERVICE = 'service'
TOPIC_SLICE      = 'slice'
TOPIC_TOPOLOGY   = 'topology'

TOPICS = {
    TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_POLICY, TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY
}

CONSUME_TIMEOUT = 0.5 # seconds

class Message(NamedTuple):
+21 −3
Changes for src/common/tools/context_queries/Topology.py: 21 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@
import grpc, logging
from typing import List, Optional
from common.Constants import DEFAULT_CONTEXT_NAME
from common.proto.context_pb2 import ContextId, Topology, TopologyId
from common.proto.context_pb2 import ContextId, Topology, TopologyDetails, TopologyId
from common.tools.object_factory.Context import json_context_id
from common.tools.object_factory.Topology import json_topology
from context.client.ContextClient import ContextClient
@@ -23,13 +23,13 @@ from context.client.ContextClient import ContextClient
LOGGER = logging.getLogger(__name__)

def create_topology(
    context_client : ContextClient, context_uuid : str, topology_uuid : str
    context_client : ContextClient, context_uuid : str, topology_uuid : str, name : Optional[str] = None
) -> None:
    context_id = ContextId(**json_context_id(context_uuid))
    existing_topology_ids = context_client.ListTopologyIds(context_id)
    existing_topology_uuids = {topology_id.topology_uuid.uuid for topology_id in existing_topology_ids.topology_ids}
    if topology_uuid in existing_topology_uuids: return
    context_client.SetTopology(Topology(**json_topology(topology_uuid, context_id=context_id)))
    context_client.SetTopology(Topology(**json_topology(topology_uuid, context_id=context_id, name=name)))

def create_missing_topologies(
    context_client : ContextClient, context_id : ContextId, topology_uuids : List[str]
@@ -61,3 +61,21 @@ def get_topology(
    except grpc.RpcError:
        #LOGGER.exception('Unable to get topology({:s} / {:s})'.format(str(context_uuid), str(topology_uuid)))
        return None

def get_topology_details(
        context_client : ContextClient, topology_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME,
        rw_copy : bool = False
    ) -> Optional[Topology]:
    try:
        # pylint: disable=no-member
        topology_id = TopologyId()
        topology_id.context_id.context_uuid.uuid = context_uuid
        topology_id.topology_uuid.uuid = topology_uuid
        ro_topology_details = context_client.GetTopologyDetails(topology_id)
        if not rw_copy: return ro_topology_details
        rw_topology_details = TopologyDetails()
        rw_topology_details.CopyFrom(ro_topology_details)
        return rw_topology_details
    except grpc.RpcError:
        #LOGGER.exception('Unable to get topology({:s} / {:s})'.format(str(context_uuid), str(topology_uuid)))
        return None
+2 −2
Changes for src/context/service/database/models/TopologyModel.py: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ class TopologyDeviceModel(_Base):
    topology_uuid = Column(ForeignKey('topology.topology_uuid', ondelete='RESTRICT'), primary_key=True, index=True)
    device_uuid   = Column(ForeignKey('device.device_uuid',     ondelete='CASCADE' ), primary_key=True, index=True)

    #topology = relationship('TopologyModel', lazy='selectin') # back_populates='topology_devices'
    topology = relationship('TopologyModel', lazy='selectin', viewonly=True) # back_populates='topology_devices'
    device   = relationship('DeviceModel',   lazy='selectin') # back_populates='topology_devices'

class TopologyLinkModel(_Base):
@@ -76,5 +76,5 @@ class TopologyLinkModel(_Base):
    topology_uuid = Column(ForeignKey('topology.topology_uuid', ondelete='RESTRICT'), primary_key=True, index=True)
    link_uuid     = Column(ForeignKey('link.link_uuid',         ondelete='CASCADE' ), primary_key=True, index=True)

    #topology = relationship('TopologyModel', lazy='selectin') # back_populates='topology_links'
    topology = relationship('TopologyModel', lazy='selectin', viewonly=True) # back_populates='topology_links'
    link     = relationship('LinkModel',     lazy='selectin') # back_populates='topology_links'
+22 −11
Changes for src/context/service/database/Connection.py: 22 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -19,7 +19,9 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, selectinload, sessionmaker
from sqlalchemy_cockroachdb import run_transaction
from typing import Dict, List, Optional, Tuple
from common.proto.context_pb2 import Connection, ConnectionId, ServiceId
from common.proto.context_pb2 import (
    Connection, ConnectionId, ConnectionIdList, ConnectionList, Empty, EventTypeEnum, ServiceId)
from common.message_broker.MessageBroker import MessageBroker
from common.method_wrappers.ServiceExceptions import NotFoundException
from common.tools.grpc.Tools import grpc_message_to_json_string
from common.tools.object_factory.Connection import json_connection_id
@@ -27,17 +29,19 @@ from .models.ConnectionModel import ConnectionEndPointModel, ConnectionModel, Co
from .uuids.Connection import connection_get_uuid
from .uuids.EndPoint import endpoint_get_uuid
from .uuids.Service import service_get_uuid
from .Events import notify_event_connection

LOGGER = logging.getLogger(__name__)

def connection_list_ids(db_engine : Engine, request : ServiceId) -> List[Dict]:
def connection_list_ids(db_engine : Engine, request : ServiceId) -> ConnectionIdList:
    _,service_uuid = service_get_uuid(request, allow_random=False)
    def callback(session : Session) -> List[Dict]:
        obj_list : List[ConnectionModel] = session.query(ConnectionModel).filter_by(service_uuid=service_uuid).all()
        return [obj.dump_id() for obj in obj_list]
    return run_transaction(sessionmaker(bind=db_engine), callback)
    connection_ids = run_transaction(sessionmaker(bind=db_engine), callback)
    return ConnectionIdList(connection_ids=connection_ids)

def connection_list_objs(db_engine : Engine, request : ServiceId) -> List[Dict]:
def connection_list_objs(db_engine : Engine, request : ServiceId) -> ConnectionList:
    _,service_uuid = service_get_uuid(request, allow_random=False)
    def callback(session : Session) -> List[Dict]:
        obj_list : List[ConnectionModel] = session.query(ConnectionModel)\
@@ -46,9 +50,10 @@ def connection_list_objs(db_engine : Engine, request : ServiceId) -> List[Dict]:
            .options(selectinload(ConnectionModel.connection_subservices))\
            .filter_by(service_uuid=service_uuid).all()
        return [obj.dump() for obj in obj_list]
    return run_transaction(sessionmaker(bind=db_engine), callback)
    connections = run_transaction(sessionmaker(bind=db_engine), callback)
    return ConnectionList(connections=connections)

def connection_get(db_engine : Engine, request : ConnectionId) -> Dict:
def connection_get(db_engine : Engine, request : ConnectionId) -> Connection:
    connection_uuid = connection_get_uuid(request, allow_random=False)
    def callback(session : Session) -> Optional[Dict]:
        obj : Optional[ConnectionModel] = session.query(ConnectionModel)\
@@ -62,9 +67,9 @@ def connection_get(db_engine : Engine, request : ConnectionId) -> Dict:
        raise NotFoundException('Connection', request.connection_uuid.uuid, extra_details=[
            'connection_uuid generated was: {:s}'.format(connection_uuid),
        ])
    return obj
    return Connection(**obj)

def connection_set(db_engine : Engine, request : Connection) -> Tuple[Dict, bool]:
def connection_set(db_engine : Engine, messagebroker : MessageBroker, request : Connection) -> ConnectionId:
    connection_uuid = connection_get_uuid(request.connection_id, allow_random=True)
    _,service_uuid = service_get_uuid(request.service_id, allow_random=False)
    settings = grpc_message_to_json_string(request.settings),
@@ -143,12 +148,18 @@ def connection_set(db_engine : Engine, request : Connection) -> Tuple[Dict, bool
        return updated

    updated = run_transaction(sessionmaker(bind=db_engine), callback)
    return json_connection_id(connection_uuid),updated
    connection_id = json_connection_id(connection_uuid)
    event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE
    notify_event_connection(messagebroker, event_type, connection_id)
    return ConnectionId(**connection_id)

def connection_delete(db_engine : Engine, request : ConnectionId) -> Tuple[Dict, bool]:
def connection_delete(db_engine : Engine, messagebroker : MessageBroker, request : ConnectionId) -> Tuple[Dict, bool]:
    connection_uuid = connection_get_uuid(request, allow_random=False)
    def callback(session : Session) -> bool:
        num_deleted = session.query(ConnectionModel).filter_by(connection_uuid=connection_uuid).delete()
        return num_deleted > 0
    deleted = run_transaction(sessionmaker(bind=db_engine), callback)
    return json_connection_id(connection_uuid),deleted
    connection_id = json_connection_id(connection_uuid)
    if deleted:
        notify_event_connection(messagebroker, EventTypeEnum.EVENTTYPE_REMOVE, connection_id)
    return Empty()
Loading
Loading