Commit 97248922 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Dlt component:

- updated manifest file with gateway container port
- corrected behaviour of GetFromDlt method in MockDltGateway
- minor formatting in MockDltGateway
- implemented first version of DltConnector's Event Dispatcher (to be tested)
- moved own-domain-uuid finder to separate class in DltConnector
- implemented method to identify empty/not-found records
parent f6b780b6
Loading
Loading
Loading
Loading
+2 −0
Changes for manifests/dltservice.yaml: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -56,6 +56,8 @@ spec:
#      - name: gateway
#        image: registry.gitlab.com/teraflow-h2020/controller/dlt-gateway:latest
#        imagePullPolicy: Always
#        ports:
#        - containerPort: 50051
#        #readinessProbe:
#        #  httpGet:
#        #    path: /health
+6 −9
Changes for src/common/tests/MockServicerImpl_DltGateway.py: 6 added lines, 9 removed lines.
Original line number Diff line number Diff line
@@ -47,16 +47,12 @@ class MockServicerImpl_DltGateway(DltGatewayServiceServicer):
        self.msg_broker = MockMessageBroker()
        LOGGER.info('[__init__] Servicer Created')

    def __get_record(self, record_id : DltRecordId, should_exist : bool) -> Optional[Dict]:
    def __get_record(self, record_id : DltRecordId) -> Optional[Dict]:
        domain_uuid, record_uuid = record_id.domain_uuid.uuid, record_id.record_uuid.uuid
        str_type = DltRecordTypeEnum.Name(record_id.type).upper().replace('DLTRECORDTYPE_', '')
        records_domain : Dict[str, Dict] = self.records.setdefault(domain_uuid, {})
        records_type   : Dict[str, Dict] = records_domain.setdefault(str_type, {})
        record         : Optional[Dict] = records_type.get(record_uuid)
        if should_exist and record is None:
            raise DoesNotExistException(MSG_NOT_EXISTS.format(domain_uuid, str_type, record_uuid))
        elif not should_exist and record is not None:
            raise AlreadyExistsException(MSG_ALREADY_EXISTS.format(domain_uuid, str_type, record_uuid))
        return record

    def __set_record(self, record_id : DltRecordId, should_exist : bool, data_json : str) -> None:
@@ -69,7 +65,7 @@ class MockServicerImpl_DltGateway(DltGatewayServiceServicer):
            raise DoesNotExistException(MSG_NOT_EXISTS.format(domain_uuid, str_type, record_uuid))
        elif not should_exist and record is not None:
            raise AlreadyExistsException(MSG_ALREADY_EXISTS.format(domain_uuid, str_type, record_uuid))
        records_type[record_uuid] = json.loads(data_json)
        records_type[record_uuid] = data_json

    def __del_record(self, record_id : DltRecordId) -> None:
        domain_uuid, record_uuid = record_id.domain_uuid.uuid, record_id.record_uuid.uuid
@@ -103,9 +99,9 @@ class MockServicerImpl_DltGateway(DltGatewayServiceServicer):
        try:
            operation : DltRecordOperationEnum = request.operation
            if operation == DLTRECORDOPERATION_ADD:
                self.__set_record(record_id, False, request.data_json)
                self.__set_record(record_id, False, json.loads(request.data_json))
            elif operation == DLTRECORDOPERATION_UPDATE:
                self.__set_record(record_id, True, request.data_json)
                self.__set_record(record_id, True, json.loads(request.data_json))
            elif operation == DLTRECORDOPERATION_DELETE:
                self.__del_record(record_id)
            else:
@@ -121,8 +117,9 @@ class MockServicerImpl_DltGateway(DltGatewayServiceServicer):

    def GetFromDlt(self, request : DltRecordId, context : grpc.ServicerContext) -> DltRecord:
        LOGGER.info('[GetFromDlt] request={:s}'.format(grpc_message_to_json_string(request)))
        record = self.__get_record(request, True)
        record = self.__get_record(request)
        response = DltRecord()
        if record is not None:
            response.record_id.CopyFrom(request) # pylint: disable=no-member
            response.operation = DLTRECORDOPERATION_UNDEFINED
            response.data_json = json.dumps(record, sort_keys=True)
+25 −2
Changes for src/dlt/connector/main_test.py: 25 added lines, 2 removed lines.
Original line number Diff line number Diff line
# pip install grpcio==1.47.0 grpcio-tools==1.47.0 protobuf==3.20.1
# PYTHONPATH=./src python
# PYTHONPATH=/home/cttc/teraflow/src python -m dlt.connector.main_test

import logging, sys, time
from common.proto.dlt_gateway_pb2 import DLTRECORDOPERATION_ADD, DLTRECORDOPERATION_UPDATE, DLTRECORDTYPE_DEVICE, DltRecord
from common.proto.dlt_gateway_pb2 import (
    DLTRECORDOPERATION_ADD, DLTRECORDOPERATION_UNDEFINED, DLTRECORDOPERATION_UPDATE, DLTRECORDTYPE_DEVICE,
    DLTRECORDTYPE_UNDEFINED, DltRecord, DltRecordId)
from common.tools.object_factory.Device import json_device
from common.tools.grpc.Tools import grpc_message_to_json_string
from src.common.proto.context_pb2 import DEVICEOPERATIONALSTATUS_ENABLED, Device
@@ -12,13 +15,33 @@ from .client.DltEventsCollector import DltEventsCollector
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)

DLT_GATEWAY_HOST = '127.0.0.1'
DLT_GATEWAY_PORT = 30551 #50051

def record_found(record : DltRecord) -> bool:
    found = True
    found = found and (len(record.record_id.domain_uuid.uuid) > 0)
    found = found and (record.record_id.type != DLTRECORDTYPE_UNDEFINED)
    found = found and (len(record.record_id.record_uuid.uuid) > 0)
    #found = found and (record.operation != DLTRECORDOPERATION_UNDEFINED)
    found = found and (len(record.data_json) > 0)
    return found

def main():
    dltgateway_client = DltGatewayClient(host='127.0.0.1', port=50051)
    dltgateway_client = DltGatewayClient(host=DLT_GATEWAY_HOST, port=DLT_GATEWAY_PORT)
    dltgateway_collector = DltEventsCollector(dltgateway_client, log_events_received=True)
    dltgateway_collector.start()

    time.sleep(3)

    # Check record exists
    dri = DltRecordId()
    dri.domain_uuid.uuid = 'non-existing-domain'
    dri.record_uuid.uuid = 'non-existing-record'
    dri.type = DLTRECORDTYPE_DEVICE
    reply = dltgateway_client.GetFromDlt(dri)
    assert not record_found(reply), 'Record should not exist'

    device = Device(**json_device('dev-1', 'packet-router', DEVICEOPERATIONALSTATUS_ENABLED))

    r2dlt_req = DltRecord()
+6 −20
Changes for src/dlt/connector/service/DltConnectorServiceServicerImpl.py: 6 added lines, 20 removed lines.
Original line number Diff line number Diff line
@@ -22,6 +22,8 @@ from common.proto.dlt_connector_pb2_grpc import DltConnectorServiceServicer
from common.tools.grpc.Tools import grpc_message_to_json_string
from context.client.ContextClient import ContextClient
from dlt.connector.client.DltGatewayClient import DltGatewayClient
from .tools.Checkers import record_exists
from .tools.OwnDomainFinder import OwnDomainFinder

LOGGER = logging.getLogger(__name__)

@@ -37,26 +39,9 @@ METRICS = create_metrics(SERVICE_NAME, METHOD_NAMES)
class DltConnectorServiceServicerImpl(DltConnectorServiceServicer):
    def __init__(self):
        LOGGER.debug('Creating Servicer...')
        self._own_domain_uuid : Optional[str] = None
        self._own_domain_finder = OwnDomainFinder()
        LOGGER.debug('Servicer Created')

    @property
    def own_domain_id(self) -> str:
        if self._own_domain_uuid is not None: return self._own_domain_uuid

        context_client = ContextClient()
        existing_context_ids = context_client.ListContextIds(Empty())
        existing_context_uuids = {context_id.context_uuid.uuid for context_id in existing_context_ids.context_ids}

        # Detect local context name (will be used as abstracted device name); exclude DEFAULT_CONTEXT_UUID
        existing_non_admin_context_uuids = copy.deepcopy(existing_context_uuids)
        existing_non_admin_context_uuids.discard(DEFAULT_CONTEXT_UUID)
        if len(existing_non_admin_context_uuids) != 1:
            MSG = 'Unable to identify own domain name. Existing Contexts({:s})'
            raise Exception(MSG.format(str(existing_context_uuids)))
        self._own_domain_uuid = existing_non_admin_context_uuids.pop()
        return self._own_domain_uuid

    @safe_and_metered_rpc_method(METRICS, LOGGER)
    def RecordAll(self, request : Empty, context : grpc.ServicerContext) -> Empty:
        return Empty()
@@ -73,7 +58,7 @@ class DltConnectorServiceServicerImpl(DltConnectorServiceServicer):
        dltgateway_client = DltGatewayClient()

        dlt_record_id = DltRecordId()
        dlt_record_id.domain_uuid.uuid = self.own_domain_id
        dlt_record_id.domain_uuid.uuid = self._own_domain_finder.own_domain_uuid
        dlt_record_id.type             = DltRecordTypeEnum.DLTRECORDTYPE_DEVICE
        dlt_record_id.record_uuid.uuid = device.device_id.device_uuid.uuid

@@ -81,7 +66,8 @@ class DltConnectorServiceServicerImpl(DltConnectorServiceServicer):
        dlt_record = dltgateway_client.GetFromDlt(dlt_record_id)
        LOGGER.info('[RecordDevice] recv dlt_record = {:s}'.format(grpc_message_to_json_string(dlt_record)))

        exists = False
        exists = record_exists(dlt_record)
        LOGGER.info('[RecordDevice] exists = {:s}'.format(str(exists)))

        dlt_record = DltRecord()
        dlt_record.record_id.CopyFrom(dlt_record_id)
+6 −6
Changes for src/dlt/connector/service/__main__.py: 6 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -18,8 +18,7 @@ from common.Constants import ServiceNameEnum
from common.Settings import (
    ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port,
    wait_for_environment_variables)
from dlt.connector.client.DltEventsCollector import DltEventsCollector
from dlt.connector.client.DltGatewayClient import DltGatewayClient
from .event_dispatcher.DltEventDispatcher import DltEventDispatcher
from .DltConnectorService import DltConnectorService

terminate = threading.Event()
@@ -50,9 +49,9 @@ def main():
    metrics_port = get_metrics_port()
    start_http_server(metrics_port)

    dlt_gateway_client = DltGatewayClient()
    dlt_events_collector = DltEventsCollector(dlt_gateway_client, log_events_received=True)
    dlt_events_collector.start()
    # Starting DLT event dispatcher
    event_dispatcher = DltEventDispatcher()
    event_dispatcher.start()

    # Starting DLT connector service
    grpc_service = DltConnectorService()
@@ -62,8 +61,9 @@ def main():
    while not terminate.wait(timeout=0.1): pass

    LOGGER.info('Terminating...')
    dlt_events_collector.stop()
    grpc_service.stop()
    event_dispatcher.stop()
    event_dispatcher.join()

    LOGGER.info('Bye')
    return 0
Loading