DLT component (and related) improvements

Common:

  • Extended common/object_factory
  • Added DLT_GATEWAY ServiceName for test and debug purposes
  • Implemented MockMessageBroker for test and debug purposes
  • Extended MockServicerImpl_Context with MockMessageBroker and refactored code
  • Implemented MockServicerImpl_DltGateway
  • Extended deploy.sh script to support DLT components
  • Removed unneeded genproto.sh script

Context:

  • Improved Context/EventsCollector generic client

DLT:

  • Separated DLT component into Connector and Gateway
  • Implemented MockBockchain component exposing DLT Gateway API to facilitate debugging
  • Implemented/Extended/Corrected Dockerfiles, manifests, etc. for DLT components
  • Implemented preliminary version of CI/CD pipeline definition for DLT (still not integrated in pipeline)

DLT Connector:

  • Implemented DltConnectorClient
  • Implemented DltEventsCollector
  • Implemented DltGatewayClient
  • Initial (incomplete) implementation of DltConnector
  • Initial (incomplete) unitary test for DltConnector

DLT Gateway:

  • Extended logic of DltGateway
Edited by Lluis Gifre Renom

Merge request reports

Loading
+86 −0
Changes for manifests/dltservice.yaml: 86 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: dltservice
spec:
  selector:
    matchLabels:
      app: dltservice
  template:
    metadata:
      labels:
        app: dltservice
    spec:
      terminationGracePeriodSeconds: 5
      containers:
      - name: connector
        image: registry.gitlab.com/teraflow-h2020/controller/dlt-connector:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
        env:
        - name: LOG_LEVEL
          value: "INFO"
        readinessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:8080"]
        livenessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:8080"]
        resources:
          requests:
            cpu: 250m
            memory: 512Mi
          limits:
            cpu: 700m
            memory: 1024Mi
      - name: gateway
        image: registry.gitlab.com/teraflow-h2020/controller/dlt-gateway:latest
        imagePullPolicy: Always
        #readinessProbe:
        #  httpGet:
        #    path: /health
        #    port: 8081
        #  initialDelaySeconds: 5
        #  timeoutSeconds: 5
        #livenessProbe:
        #  httpGet:
        #    path: /health
        #    port: 8081
        #  initialDelaySeconds: 5
        #  timeoutSeconds: 5
        resources:
          requests:
            cpu: 250m
            memory: 512Mi
          limits:
            cpu: 700m
            memory: 1024Mi
---
apiVersion: v1
kind: Service
metadata:
  name: dltservice
spec:
  type: ClusterIP
  selector:
    app: dltservice
  ports:
  - name: grpc
    protocol: TCP
    port: 8080
    targetPort: 8080
+64 −0
Changes for manifests/mock_blockchain.yaml: 64 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mock-blockchain
spec:
  selector:
    matchLabels:
      app: mock-blockchain
  template:
    metadata:
      labels:
        app: mock-blockchain
    spec:
      terminationGracePeriodSeconds: 5
      containers:
      - name: server
        image: registry.gitlab.com/teraflow-h2020/controller/mock_blockchain:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 50051
        env:
        - name: LOG_LEVEL
          value: "DEBUG"
        readinessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:50051"]
        livenessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:50051"]
        resources:
          requests:
            cpu: 250m
            memory: 512Mi
          limits:
            cpu: 700m
            memory: 1024Mi
---
apiVersion: v1
kind: Service
metadata:
  name: mock-blockchain
spec:
  type: ClusterIP
  selector:
    app: mock-blockchain
  ports:
  - name: grpc
    protocol: TCP
    port: 50051
    targetPort: 50051
+2 −2
Changes for proto/dlt_gateway.proto: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -21,8 +21,8 @@ service DltGatewayService {
  rpc RecordToDlt   (DltRecord                 ) returns (       DltRecordStatus  ) {}
  rpc GetFromDlt    (DltRecordId               ) returns (       DltRecord        ) {}
  rpc SubscribeToDlt(DltRecordSubscription     ) returns (stream DltRecordEvent   ) {}
  rpc GetDltStatus  (context.TeraFlowController) returns (       DltPeerStatus    ) {}  // NEC is checkig if it is possible
  rpc GetDltPeers   (context.Empty             ) returns (       DltPeerStatusList) {}  // NEC is checkig if it is possible
  rpc GetDltStatus  (context.TeraFlowController) returns (       DltPeerStatus    ) {}  // NEC is checking if it is possible
  rpc GetDltPeers   (context.Empty             ) returns (       DltPeerStatusList) {}  // NEC is checking if it is possible
}

enum DltRecordTypeEnum {
+61 −0
Changes for src/common/tests/MockMessageBroker.py: 61 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json, logging, threading, time
from queue import Queue, Empty
from typing import Dict, Iterator, NamedTuple, Set

LOGGER = logging.getLogger(__name__)
CONSUME_TIMEOUT = 0.1 # seconds

class Message(NamedTuple):
    topic: str
    content: str

class MockMessageBroker:
    def __init__(self):
        self._terminate = threading.Event()
        self._topic__to__queues : Dict[str, Set[Queue]] = {}

    def publish(self, message : Message) -> None:
        queues = self._topic__to__queues.get(message.topic, None)
        if queues is None: return
        for queue in queues: queue.put_nowait((message.topic, message.content))

    def consume(
        self, topic_names : Set[str], block : bool = True, consume_timeout : float = CONSUME_TIMEOUT
    ) -> Iterator[Message]:
        queue = Queue()
        for topic_name in topic_names:
            self._topic__to__queues.setdefault(topic_name, set()).add(queue)

        while not self._terminate.is_set():
            try:
                message = queue.get(block=block, timeout=consume_timeout)
            except Empty:
                continue
            if message is None: continue
            yield Message(*message)

        for topic_name in topic_names:
            self._topic__to__queues.get(topic_name, set()).discard(queue)

    def terminate(self):
        self._terminate.set()

def notify_event(messagebroker, topic_name, event_type, fields) -> None:
    event = {'event': {'timestamp': time.time(), 'event_type': event_type}}
    for field_name, field_value in fields.items():
        event[field_name] = field_value
    messagebroker.publish(Message(topic_name, json.dumps(event)))
+67 −26
Changes for src/common/tests/MockServicerImpl_Context.py: 67 added lines, 26 removed lines.
Original line number Diff line number Diff line
@@ -12,22 +12,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import grpc, logging
import grpc, json, logging
from typing import Any, Dict, Iterator, List
from common.proto.context_pb2 import (
    Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList,
    Context, ContextEvent, ContextId, ContextIdList, ContextList,
    Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList,
    Empty,
    Empty, EventTypeEnum,
    Link, LinkEvent, LinkId, LinkIdList, LinkList,
    Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList,
    Slice, SliceEvent, SliceId, SliceIdList, SliceList,
    Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList)
from common.proto.context_pb2_grpc import ContextServiceServicer
from common.tools.grpc.Tools import grpc_message_to_json_string
from common.tests.MockMessageBroker import MockMessageBroker, notify_event
from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string

LOGGER = logging.getLogger(__name__)

TOPIC_CONNECTION = 'connection'
TOPIC_CONTEXT    = 'context'
TOPIC_TOPOLOGY   = 'topology'
TOPIC_DEVICE     = 'device'
TOPIC_LINK       = 'link'
TOPIC_SERVICE    = 'service'
TOPIC_SLICE      = 'slice'

def get_container(database : Dict[str, Dict[str, Any]], container_name : str) -> Dict[str, Any]:
    return database.setdefault(container_name, {})

@@ -35,10 +44,15 @@ def get_entries(database : Dict[str, Dict[str, Any]], container_name : str) -> L
    container = get_container(database, container_name)
    return [container[entry_uuid] for entry_uuid in sorted(container.keys())]

def has_entry(database : Dict[str, Dict[str, Any]], container_name : str, entry_uuid : str) -> Any:
    LOGGER.debug('[has_entry] BEFORE database={:s}'.format(str(database)))
    container = get_container(database, container_name)
    return entry_uuid in container

def get_entry(
    context : grpc.ServicerContext, database : Dict[str, Dict[str, Any]], container_name : str, entry_uuid : str
) -> Any:
    LOGGER.debug('[get_entry] AFTER database={:s}'.format(str(database)))
    LOGGER.debug('[get_entry] BEFORE database={:s}'.format(str(database)))
    container = get_container(database, container_name)
    if entry_uuid not in container:
        context.abort(grpc.StatusCode.NOT_FOUND, str('{:s}({:s}) not found'.format(container_name, entry_uuid)))
@@ -64,8 +78,27 @@ class MockServicerImpl_Context(ContextServiceServicer):
    def __init__(self):
        LOGGER.info('[__init__] Creating Servicer...')
        self.database : Dict[str, Any] = {}
        self.msg_broker = MockMessageBroker()
        LOGGER.info('[__init__] Servicer Created')

    # ----- Common -----------------------------------------------------------------------------------------------------

    def _set(self, request, container_name, entry_uuid, entry_id_field_name, topic_name):
        exists = has_entry(self.database, container_name, entry_uuid)
        entry = set_entry(self.database, container_name, entry_uuid, request)
        event_type = EventTypeEnum.EVENTTYPE_UPDATE if exists else EventTypeEnum.EVENTTYPE_CREATE
        entry_id = getattr(entry, entry_id_field_name)
        dict_entry_id = grpc_message_to_json(entry_id)
        notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id})
        return entry_id

    def _del(self, request, container_name, entry_uuid, entry_id_field_name, topic_name, grpc_context):
        empty = del_entry(grpc_context, self.database, container_name, entry_uuid)
        event_type = EventTypeEnum.EVENTTYPE_REMOVE
        dict_entry_id = grpc_message_to_json(request)
        notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id})
        return empty

    # ----- Context ----------------------------------------------------------------------------------------------------

    def ListContextIds(self, request: Empty, context : grpc.ServicerContext) -> ContextIdList:
@@ -82,14 +115,15 @@ class MockServicerImpl_Context(ContextServiceServicer):

    def SetContext(self, request: Context, context : grpc.ServicerContext) -> ContextId:
        LOGGER.info('[SetContext] request={:s}'.format(grpc_message_to_json_string(request)))
        return set_entry(self.database, 'context', request.context_id.context_uuid.uuid, request).context_id
        return self._set(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT)

    def RemoveContext(self, request: ContextId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveContext] request={:s}'.format(grpc_message_to_json_string(request)))
        return del_entry(context, self.database, 'context', request.context_uuid.uuid)
        return self._del(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT, context)

    def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]:
        LOGGER.info('[GetContextEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content))


    # ----- Topology ---------------------------------------------------------------------------------------------------
@@ -112,15 +146,18 @@ class MockServicerImpl_Context(ContextServiceServicer):
    def SetTopology(self, request: Topology, context : grpc.ServicerContext) -> TopologyId:
        LOGGER.info('[SetTopology] request={:s}'.format(grpc_message_to_json_string(request)))
        container_name = 'topology[{:s}]'.format(str(request.topology_id.context_id.context_uuid.uuid))
        return set_entry(self.database, container_name, request.topology_id.topology_uuid.uuid, request).topology_id
        topology_uuid = request.topology_id.topology_uuid.uuid
        return self._set(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY)

    def RemoveTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveTopology] request={:s}'.format(grpc_message_to_json_string(request)))
        container_name = 'topology[{:s}]'.format(str(request.context_id.context_uuid.uuid))
        return del_entry(context, self.database, container_name, request.topology_uuid.uuid)
        topology_uuid = request.topology_uuid.uuid
        return self._del(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY, context)

    def GetTopologyEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[TopologyEvent]:
        LOGGER.info('[GetTopologyEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_TOPOLOGY}): yield TopologyEvent(**json.loads(message.content))


    # ----- Device -----------------------------------------------------------------------------------------------------
@@ -139,14 +176,15 @@ class MockServicerImpl_Context(ContextServiceServicer):

    def SetDevice(self, request: Context, context : grpc.ServicerContext) -> DeviceId:
        LOGGER.info('[SetDevice] request={:s}'.format(grpc_message_to_json_string(request)))
        return set_entry(self.database, 'device', request.device_id.device_uuid.uuid, request).device_id
        return self._set(request, 'device', request.device_id.device_uuid.uuid, 'device_id', TOPIC_DEVICE)

    def RemoveDevice(self, request: DeviceId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveDevice] request={:s}'.format(grpc_message_to_json_string(request)))
        return del_entry(context, self.database, 'device', request.device_uuid.uuid)
        return self._del(request, 'device', request.device_uuid.uuid, 'device_id', TOPIC_DEVICE, context)

    def GetDeviceEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[DeviceEvent]:
        LOGGER.info('[GetDeviceEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_DEVICE}): yield DeviceEvent(**json.loads(message.content))


    # ----- Link -------------------------------------------------------------------------------------------------------
@@ -165,14 +203,15 @@ class MockServicerImpl_Context(ContextServiceServicer):

    def SetLink(self, request: Context, context : grpc.ServicerContext) -> LinkId:
        LOGGER.info('[SetLink] request={:s}'.format(grpc_message_to_json_string(request)))
        return set_entry(self.database, 'link', request.link_id.link_uuid.uuid, request).link_id
        return self._set(request, 'link', request.link_id.link_uuid.uuid, 'link_id', TOPIC_LINK)

    def RemoveLink(self, request: LinkId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveLink] request={:s}'.format(grpc_message_to_json_string(request)))
        return del_entry(context, self.database, 'link', request.link_uuid.uuid)
        return self._del(request, 'link', request.link_uuid.uuid, 'link_id', TOPIC_LINK, context)

    def GetLinkEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[LinkEvent]:
        LOGGER.info('[GetLinkEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_LINK}): yield LinkEvent(**json.loads(message.content))


    # ----- Slice ------------------------------------------------------------------------------------------------------
@@ -226,17 +265,19 @@ class MockServicerImpl_Context(ContextServiceServicer):

    def SetService(self, request: Service, context : grpc.ServicerContext) -> ServiceId:
        LOGGER.info('[SetService] request={:s}'.format(grpc_message_to_json_string(request)))
        return set_entry(
            self.database, 'service[{:s}]'.format(str(request.service_id.context_id.context_uuid.uuid)),
            request.service_id.service_uuid.uuid, request).service_id
        container_name = 'service[{:s}]'.format(str(request.service_id.context_id.context_uuid.uuid))
        service_uuid = request.service_id.service_uuid.uuid
        return self._set(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE)

    def RemoveService(self, request: ServiceId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveService] request={:s}'.format(grpc_message_to_json_string(request)))
        container_name = 'service[{:s}]'.format(str(request.context_id.context_uuid.uuid))
        return del_entry(context, self.database, container_name, request.service_uuid.uuid)
        service_uuid = request.service_id.service_uuid.uuid
        return self._del(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE, context)

    def GetServiceEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ServiceEvent]:
        LOGGER.info('[GetServiceEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_SERVICE}): yield ServiceEvent(**json.loads(message.content))


    # ----- Connection -------------------------------------------------------------------------------------------------
@@ -259,21 +300,21 @@ class MockServicerImpl_Context(ContextServiceServicer):

    def SetConnection(self, request: Connection, context : grpc.ServicerContext) -> ConnectionId:
        LOGGER.info('[SetConnection] request={:s}'.format(grpc_message_to_json_string(request)))
        service_connection__container_name = 'service_connection[{:s}/{:s}]'.format(
        container_name = 'service_connection[{:s}/{:s}]'.format(
            str(request.service_id.context_id.context_uuid.uuid), str(request.service_id.service_uuid.uuid))
        set_entry(
            self.database, service_connection__container_name, request.connection_id.connection_uuid.uuid, request)
        return set_entry(
            self.database, 'connection', request.connection_id.connection_uuid.uuid, request).connection_id
        connection_uuid = request.connection_id.connection_uuid.uuid
        set_entry(self.database, container_name, connection_uuid, request)
        return self._set(request, 'connection', connection_uuid, 'connection_id', TOPIC_CONNECTION)

    def RemoveConnection(self, request: ConnectionId, context : grpc.ServicerContext) -> Empty:
        LOGGER.info('[RemoveConnection] request={:s}'.format(grpc_message_to_json_string(request)))
        connection = get_entry(context, self.database, 'connection', request.connection_uuid.uuid)
        service_id = connection.service_id
        service_connection__container_name = 'service_connection[{:s}/{:s}]'.format(
            str(service_id.context_id.context_uuid.uuid), str(service_id.service_uuid.uuid))
        del_entry(context, self.database, service_connection__container_name, request.connection_uuid.uuid)
        return del_entry(context, self.database, 'connection', request.connection_uuid.uuid)
        container_name = 'service_connection[{:s}/{:s}]'.format(
            str(connection.service_id.context_id.context_uuid.uuid), str(connection.service_id.service_uuid.uuid))
        connection_uuid = request.connection_uuid.uuid
        del_entry(context, self.database, container_name, connection_uuid)
        return self._del(request, 'connection', connection_uuid, 'connection_id', TOPIC_CONNECTION, context)

    def GetConnectionEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ConnectionEvent]:
        LOGGER.info('[GetConnectionEvents] request={:s}'.format(grpc_message_to_json_string(request)))
        for message in self.msg_broker.consume({TOPIC_CONNECTION}): yield ConnectionEvent(**json.loads(message.content))
Loading
Loading