From 205d06335c9b0eb8f3d6c5927059336483795389 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Wed, 6 Jul 2022 17:54:01 +0200 Subject: [PATCH 01/36] DLT component: - Added requirements.in - Added default config file - Created objects for the unitary test - Created scenario settings for the unitary test --- src/dlt/connector/Config.py | 14 +++++ src/dlt/connector/requirements.in | 6 +++ src/dlt/connector/tests/Objects.py | 53 +++++++++++++++++++ .../connector/tests/PrepareTestScenario.py | 50 +++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 src/dlt/connector/Config.py create mode 100644 src/dlt/connector/requirements.in create mode 100644 src/dlt/connector/tests/Objects.py create mode 100644 src/dlt/connector/tests/PrepareTestScenario.py diff --git a/src/dlt/connector/Config.py b/src/dlt/connector/Config.py new file mode 100644 index 000000000..70a332512 --- /dev/null +++ b/src/dlt/connector/Config.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/dlt/connector/requirements.in b/src/dlt/connector/requirements.in new file mode 100644 index 000000000..162ecde82 --- /dev/null +++ b/src/dlt/connector/requirements.in @@ -0,0 +1,6 @@ +grpcio==1.43.0 +grpcio-health-checking==1.43.0 +prometheus-client==0.13.0 +protobuf==3.19.3 +pytest==6.2.5 +pytest-benchmark==3.4.1 diff --git a/src/dlt/connector/tests/Objects.py b/src/dlt/connector/tests/Objects.py new file mode 100644 index 000000000..cb3baf9c9 --- /dev/null +++ b/src/dlt/connector/tests/Objects.py @@ -0,0 +1,53 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID +from common.tools.object_factory.Context import json_context, json_context_id +from common.tools.object_factory.Device import json_device_emulated_packet_router_disabled, json_device_id +from common.tools.object_factory.EndPoint import json_endpoints +from common.tools.object_factory.Link import compose_link +from common.tools.object_factory.Topology import json_topology, json_topology_id + +def compose_device( + device_uuid, endpoint_uuids, endpoint_type='copper', endpoint_topology_id=None, endpoint_sample_types=[] +): + device_id = json_device_id(device_uuid) + endpoints = [(endpoint_uuid, endpoint_type, endpoint_sample_types) for endpoint_uuid in endpoint_uuids] + endpoints = json_endpoints(device_id, endpoints, topology_id=endpoint_topology_id) + device = json_device_emulated_packet_router_disabled(device_uuid, endpoints=endpoints) + return device_id, endpoints, device + +# ----- Context -------------------------------------------------------------------------------------------------------- +CONTEXT_ADMIN_ID = json_context_id(DEFAULT_CONTEXT_UUID) +CONTEXT_ADMIN = json_context(DEFAULT_CONTEXT_UUID) + +# ----- Topology ------------------------------------------------------------------------------------------------------- +TOPOLOGY_ADMIN_ID = json_topology_id(DEFAULT_TOPOLOGY_UUID, context_id=CONTEXT_ADMIN_ID) +TOPOLOGY_ADMIN = json_topology(DEFAULT_TOPOLOGY_UUID, context_id=CONTEXT_ADMIN_ID) + +# ----- Devices -------------------------------------------------------------------------------------------------------- +DEVICE_DEV1_ID, DEVICE_DEV1_ENDPOINTS, DEVICE_DEV1 = compose_device('DEV1', ['1', '2']) +DEVICE_DEV2_ID, DEVICE_DEV2_ENDPOINTS, DEVICE_DEV2 = compose_device('DEV2', ['1', '2']) +DEVICE_DEV3_ID, DEVICE_DEV3_ENDPOINTS, DEVICE_DEV3 = compose_device('DEV3', ['1', '2']) + +# ----- Links ---------------------------------------------------------------------------------------------------------- +LINK_DEV1_DEV2_ID, LINK_DEV1_DEV2 = compose_link(DEVICE_DEV1_ENDPOINTS[0], DEVICE_DEV2_ENDPOINTS[0]) +LINK_DEV1_DEV3_ID, LINK_DEV1_DEV3 = compose_link(DEVICE_DEV1_ENDPOINTS[1], DEVICE_DEV3_ENDPOINTS[0]) +LINK_DEV2_DEV3_ID, LINK_DEV2_DEV3 = compose_link(DEVICE_DEV2_ENDPOINTS[1], DEVICE_DEV3_ENDPOINTS[1]) + +# ----- Containers ----------------------------------------------------------------------------------------------------- +CONTEXTS = [CONTEXT_ADMIN] +TOPOLOGIES = [TOPOLOGY_ADMIN] +DEVICES = [DEVICE_DEV1, DEVICE_DEV2, DEVICE_DEV3] +LINKS = [LINK_DEV1_DEV2, LINK_DEV1_DEV3, LINK_DEV2_DEV3] diff --git a/src/dlt/connector/tests/PrepareTestScenario.py b/src/dlt/connector/tests/PrepareTestScenario.py new file mode 100644 index 000000000..271deb72a --- /dev/null +++ b/src/dlt/connector/tests/PrepareTestScenario.py @@ -0,0 +1,50 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, pytest +from typing import Tuple +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) +from common.orm.Database import Database +from common.orm.Factory import get_database_backend, BackendEnum as DatabaseBackendEnum +from common.message_broker.Factory import get_messagebroker_backend, BackendEnum as MessageBrokerBackendEnum +from common.message_broker.MessageBroker import MessageBroker +from context.client.ContextClient import ContextClient +from context.service.grpc_server.ContextService import ContextService + +LOCAL_HOST = '127.0.0.1' +GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT) # avoid privileged ports +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT) + +@pytest.fixture(scope='session') +def context_db_mb() -> Tuple[Database, MessageBroker]: + _database = Database(get_database_backend(backend=DatabaseBackendEnum.INMEMORY)) + _message_broker = MessageBroker(get_messagebroker_backend(backend=MessageBrokerBackendEnum.INMEMORY)) + yield _database, _message_broker + _message_broker.terminate() + +@pytest.fixture(scope='session') +def context_service(context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name + _service = ContextService(context_db_mb[0], context_db_mb[1]) + _service.start() + yield _service + _service.stop() + +@pytest.fixture(scope='session') +def context_client(context_service : ContextService): # pylint: disable=redefined-outer-name + _client = ContextClient() + yield _client + _client.close() -- GitLab From 402ed56b028966d168427eea2b66134e4ad0d768 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Wed, 6 Jul 2022 17:54:26 +0200 Subject: [PATCH 02/36] Context component: - improved configuration of EventsCollector --- src/context/client/EventsCollector.py | 107 +++++++++++++++++--------- 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/src/context/client/EventsCollector.py b/src/context/client/EventsCollector.py index f35b43eab..071645bd3 100644 --- a/src/context/client/EventsCollector.py +++ b/src/context/client/EventsCollector.py @@ -22,26 +22,57 @@ LOGGER.setLevel(logging.DEBUG) class EventsCollector: def __init__( - self, context_client_grpc : ContextClient, log_events_received=False + self, context_client : ContextClient, + log_events_received : bool = False, + activate_context_collector : bool = True, + activate_topology_collector : bool = True, + activate_device_collector : bool = True, + activate_link_collector : bool = True, + activate_service_collector : bool = True, + activate_slice_collector : bool = True, + activate_connection_collector : bool = True, + ) -> None: self._events_queue = queue.Queue() self._log_events_received = log_events_received - self._context_stream = context_client_grpc.GetContextEvents(Empty()) - self._topology_stream = context_client_grpc.GetTopologyEvents(Empty()) - self._device_stream = context_client_grpc.GetDeviceEvents(Empty()) - self._link_stream = context_client_grpc.GetLinkEvents(Empty()) - self._service_stream = context_client_grpc.GetServiceEvents(Empty()) - self._slice_stream = context_client_grpc.GetSliceEvents(Empty()) - self._connection_stream = context_client_grpc.GetConnectionEvents(Empty()) - - self._context_thread = threading.Thread(target=self._collect, args=(self._context_stream ,), daemon=False) - self._topology_thread = threading.Thread(target=self._collect, args=(self._topology_stream ,), daemon=False) - self._device_thread = threading.Thread(target=self._collect, args=(self._device_stream ,), daemon=False) - self._link_thread = threading.Thread(target=self._collect, args=(self._link_stream ,), daemon=False) - self._service_thread = threading.Thread(target=self._collect, args=(self._service_stream ,), daemon=False) - self._slice_thread = threading.Thread(target=self._collect, args=(self._slice_stream ,), daemon=False) - self._connection_thread = threading.Thread(target=self._collect, args=(self._connection_stream,), daemon=False) + self._context_stream, self._context_thread = None, None + if activate_context_collector: + self._context_stream = context_client.GetContextEvents(Empty()) + self._context_thread = self._create_collector_thread(self._context_stream) + + self._topology_stream, self._topology_thread = None, None + if activate_topology_collector: + self._topology_stream = context_client.GetTopologyEvents(Empty()) + self._topology_thread = self._create_collector_thread(self._topology_stream) + + self._device_stream, self._device_thread = None, None + if activate_device_collector: + self._device_stream = context_client.GetDeviceEvents(Empty()) + self._device_thread = self._create_collector_thread(self._device_stream) + + self._link_stream, self._link_thread = None, None + if activate_link_collector: + self._link_stream = context_client.GetLinkEvents(Empty()) + self._link_thread = self._create_collector_thread(self._link_stream) + + self._service_stream, self._service_thread = None, None + if activate_service_collector: + self._service_stream = context_client.GetServiceEvents(Empty()) + self._service_thread = self._create_collector_thread(self._service_stream) + + self._slice_stream, self._slice_thread = None, None + if activate_slice_collector: + self._slice_stream = context_client.GetSliceEvents(Empty()) + self._slice_thread = self._create_collector_thread(self._slice_stream) + + self._connection_stream, self._connection_thread = None, None + if activate_connection_collector: + self._connection_stream = context_client.GetConnectionEvents(Empty()) + self._connection_thread = self._create_collector_thread(self._connection_stream) + + def _create_collector_thread(self, stream, as_daemon : bool = False): + return threading.Thread(target=self._collect, args=(stream,), daemon=as_daemon) def _collect(self, events_stream) -> None: try: @@ -54,13 +85,13 @@ class EventsCollector: raise # pragma: no cover def start(self): - self._context_thread.start() - self._topology_thread.start() - self._device_thread.start() - self._link_thread.start() - self._service_thread.start() - self._slice_thread.start() - self._connection_thread.start() + if self._context_thread is not None: self._context_thread.start() + if self._topology_thread is not None: self._topology_thread.start() + if self._device_thread is not None: self._device_thread.start() + if self._link_thread is not None: self._link_thread.start() + if self._service_thread is not None: self._service_thread.start() + if self._slice_thread is not None: self._slice_thread.start() + if self._connection_thread is not None: self._connection_thread.start() def get_event(self, block : bool = True, timeout : float = 0.1): try: @@ -83,18 +114,18 @@ class EventsCollector: return sorted(events, key=lambda e: e.event.timestamp) def stop(self): - self._context_stream.cancel() - self._topology_stream.cancel() - self._device_stream.cancel() - self._link_stream.cancel() - self._service_stream.cancel() - self._slice_stream.cancel() - self._connection_stream.cancel() - - self._context_thread.join() - self._topology_thread.join() - self._device_thread.join() - self._link_thread.join() - self._service_thread.join() - self._slice_thread.join() - self._connection_thread.join() + if self._context_stream is not None: self._context_stream.cancel() + if self._topology_stream is not None: self._topology_stream.cancel() + if self._device_stream is not None: self._device_stream.cancel() + if self._link_stream is not None: self._link_stream.cancel() + if self._service_stream is not None: self._service_stream.cancel() + if self._slice_stream is not None: self._slice_stream.cancel() + if self._connection_stream is not None: self._connection_stream.cancel() + + if self._context_thread is not None: self._context_thread.join() + if self._topology_thread is not None: self._topology_thread.join() + if self._device_thread is not None: self._device_thread.join() + if self._link_thread is not None: self._link_thread.join() + if self._service_thread is not None: self._service_thread.join() + if self._slice_thread is not None: self._slice_thread.join() + if self._connection_thread is not None: self._connection_thread.join() -- GitLab From a173e1ab173f53c0eef1ae9c64be60f44c39ef1c Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Wed, 6 Jul 2022 17:54:58 +0200 Subject: [PATCH 03/36] Common: - improved object factory tools for links --- src/common/tools/object_factory/Link.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/common/tools/object_factory/Link.py b/src/common/tools/object_factory/Link.py index 624cbb8dc..9519335ca 100644 --- a/src/common/tools/object_factory/Link.py +++ b/src/common/tools/object_factory/Link.py @@ -13,15 +13,21 @@ # limitations under the License. import copy -from typing import Dict, List +from typing import Dict, List, Tuple -def get_link_uuid(a_device_id : Dict, a_endpoint_id : Dict, z_device_id : Dict, z_endpoint_id : Dict) -> str: +def get_link_uuid(a_endpoint_id : Dict, z_endpoint_id : Dict) -> str: return '{:s}/{:s}=={:s}/{:s}'.format( - a_device_id['device_uuid']['uuid'], a_endpoint_id['endpoint_uuid']['uuid'], - z_device_id['device_uuid']['uuid'], z_endpoint_id['endpoint_uuid']['uuid']) + a_endpoint_id['device_id']['device_uuid']['uuid'], a_endpoint_id['endpoint_uuid']['uuid'], + a_endpoint_id['device_id']['device_uuid']['uuid'], z_endpoint_id['endpoint_uuid']['uuid']) -def json_link_id(link_uuid : str): +def json_link_id(link_uuid : str) -> Dict: return {'link_uuid': {'uuid': link_uuid}} -def json_link(link_uuid : str, endpoint_ids : List[Dict]): +def json_link(link_uuid : str, endpoint_ids : List[Dict]) -> Dict: return {'link_id': json_link_id(link_uuid), 'link_endpoint_ids': copy.deepcopy(endpoint_ids)} + +def compose_link(endpoint_a, endpoint_z) -> Tuple[Dict, Dict]: + link_uuid = get_link_uuid(endpoint_a['endpoint_id'], endpoint_z['endpoint_id']) + link_id = json_link_id(link_uuid) + link = json_link(link_uuid, [endpoint_a['endpoint_id'], endpoint_z['endpoint_id']]) + return link_id, link -- GitLab From f30a22a7ef2c2b54679883b61b01aa8ecc92df6c Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Thu, 14 Jul 2022 18:16:58 +0200 Subject: [PATCH 04/36] Intermediate backup of DLT connector (not functional) --- src/common/tests/MockMessageBroker.py | 61 +++++++ src/common/tests/MockServicerImpl_Context.py | 103 ++++++++--- .../tests/MockServicerImpl_Service copy.py | 108 +++++++++++ src/dlt/connector/.gitlab-ci.yml | 74 ++++++++ src/dlt/connector/Dockerfile | 38 ++++ src/dlt/connector/client/DltClient.py | 171 ++++++++++++++++++ src/dlt/connector/service/DltConnector.py | 51 ++++++ .../connector/service/DltConnectorService.py | 28 +++ .../DltConnectorServiceServicerImpl.py | 33 ++++ src/dlt/connector/service/__main__.py | 65 +++++++ src/dlt/connector/tests/test_unitary.py | 49 +++++ 11 files changed, 752 insertions(+), 29 deletions(-) create mode 100644 src/common/tests/MockMessageBroker.py create mode 100644 src/common/tests/MockServicerImpl_Service copy.py create mode 100644 src/dlt/connector/.gitlab-ci.yml create mode 100644 src/dlt/connector/Dockerfile create mode 100644 src/dlt/connector/client/DltClient.py create mode 100644 src/dlt/connector/service/DltConnector.py create mode 100644 src/dlt/connector/service/DltConnectorService.py create mode 100644 src/dlt/connector/service/DltConnectorServiceServicerImpl.py create mode 100644 src/dlt/connector/service/__main__.py create mode 100644 src/dlt/connector/tests/test_unitary.py diff --git a/src/common/tests/MockMessageBroker.py b/src/common/tests/MockMessageBroker.py new file mode 100644 index 000000000..851c06766 --- /dev/null +++ b/src/common/tests/MockMessageBroker.py @@ -0,0 +1,61 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, logging, threading, time +from queue import Queue, Empty +from typing import Dict, Iterator, NamedTuple, Set + +LOGGER = logging.getLogger(__name__) +CONSUME_TIMEOUT = 0.1 # seconds + +class Message(NamedTuple): + topic: str + content: str + +class MockMessageBroker: + def __init__(self): + self._terminate = threading.Event() + self._topic__to__queues : Dict[str, Set[Queue]] = {} + + def publish(self, message : Message) -> None: + queues = self._topic__to__queues.get(message.topic, None) + if queues is None: return + for queue in queues: queue.put_nowait((message.topic, message.content)) + + def consume( + self, topic_names : Set[str], block : bool = True, consume_timeout : float = CONSUME_TIMEOUT + ) -> Iterator[Message]: + queue = Queue() + for topic_name in topic_names: + self._topic__to__queues.setdefault(topic_name, set()).add(queue) + + while not self._terminate.is_set(): + try: + message = queue.get(block=block, timeout=consume_timeout) + except Empty: + continue + if message is None: continue + yield Message(*message) + + for topic_name in topic_names: + self._topic__to__queues.get(topic_name, set()).discard(queue) + + def terminate(self): + self._terminate.set() + +def notify_event(messagebroker, topic_name, event_type, fields) -> None: + event = {'event': {'timestamp': time.time(), 'event_type': event_type}} + for field_name, field_value in fields.items(): + event[field_name] = field_value + messagebroker.publish(Message(topic_name, json.dumps(event))) diff --git a/src/common/tests/MockServicerImpl_Context.py b/src/common/tests/MockServicerImpl_Context.py index 8b4560016..adb8ae360 100644 --- a/src/common/tests/MockServicerImpl_Context.py +++ b/src/common/tests/MockServicerImpl_Context.py @@ -12,18 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc, logging +import grpc, json, logging from typing import Any, Dict, Iterator, List -from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tests.MockMessageBroker import MockMessageBroker, notify_event +from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string from context.proto.context_pb2 import ( - Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList, Context, ContextEvent, ContextId, - ContextIdList, ContextList, Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList, Empty, Link, LinkEvent, - LinkId, LinkIdList, LinkList, Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList, Slice, SliceEvent, - SliceId, SliceIdList, SliceList, Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList) + Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList, + Context, ContextEvent, ContextId, ContextIdList, ContextList, + Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList, + Empty, EventTypeEnum, + Link, LinkEvent, LinkId, LinkIdList, LinkList, + Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList, + Slice, SliceEvent, SliceId, SliceIdList, SliceList, + Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList) from context.proto.context_pb2_grpc import ContextServiceServicer LOGGER = logging.getLogger(__name__) +TOPIC_CONNECTION = 'connection' +TOPIC_CONTEXT = 'context' +TOPIC_TOPOLOGY = 'topology' +TOPIC_DEVICE = 'device' +TOPIC_LINK = 'link' +TOPIC_SERVICE = 'service' +TOPIC_SLICE = 'slice' + def get_container(database : Dict[str, Dict[str, Any]], container_name : str) -> Dict[str, Any]: return database.setdefault(container_name, {}) @@ -31,10 +44,15 @@ def get_entries(database : Dict[str, Dict[str, Any]], container_name : str) -> L container = get_container(database, container_name) return [container[entry_uuid] for entry_uuid in sorted(container.keys())] +def has_entry(database : Dict[str, Dict[str, Any]], container_name : str, entry_uuid : str) -> Any: + LOGGER.debug('[has_entry] BEFORE database={:s}'.format(str(database))) + container = get_container(database, container_name) + return entry_uuid in container + def get_entry( context : grpc.ServicerContext, database : Dict[str, Dict[str, Any]], container_name : str, entry_uuid : str ) -> Any: - LOGGER.debug('[get_entry] AFTER database={:s}'.format(str(database))) + LOGGER.debug('[get_entry] BEFORE database={:s}'.format(str(database))) container = get_container(database, container_name) if entry_uuid not in container: context.abort(grpc.StatusCode.NOT_FOUND, str('{:s}({:s}) not found'.format(container_name, entry_uuid))) @@ -60,8 +78,27 @@ class MockServicerImpl_Context(ContextServiceServicer): def __init__(self): LOGGER.info('[__init__] Creating Servicer...') self.database : Dict[str, Any] = {} + self.msg_broker = MockMessageBroker() LOGGER.info('[__init__] Servicer Created') + # ----- Common ----------------------------------------------------------------------------------------------------- + + def _set(self, request, container_name, entry_uuid, entry_id_field_name, topic_name): + exists = has_entry(self.database, container_name, entry_uuid) + entry = set_entry(self.database, container_name, entry_uuid, request) + event_type = EventTypeEnum.EVENTTYPE_UPDATE if exists else EventTypeEnum.EVENTTYPE_CREATE + entry_id = getattr(entry, entry_id_field_name) + dict_entry_id = grpc_message_to_json(entry_id) + notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) + return entry_id + + def _del(self, request, container_name, entry_uuid, entry_id_field_name, topic_name, grpc_context): + empty = del_entry(grpc_context, self.database, container_name, entry_uuid) + event_type = EventTypeEnum.EVENTTYPE_REMOVE + dict_entry_id = grpc_message_to_json(request) + notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) + return empty + # ----- Context ---------------------------------------------------------------------------------------------------- def ListContextIds(self, request: Empty, context : grpc.ServicerContext) -> ContextIdList: @@ -78,14 +115,15 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetContext(self, request: Context, context : grpc.ServicerContext) -> ContextId: LOGGER.info('[SetContext] request={:s}'.format(grpc_message_to_json_string(request))) - return set_entry(self.database, 'context', request.context_id.context_uuid.uuid, request).context_id + return self._set(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT) def RemoveContext(self, request: ContextId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveContext] request={:s}'.format(grpc_message_to_json_string(request))) - return del_entry(context, self.database, 'context', request.context_uuid.uuid) + return self._del(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT, context) def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]: LOGGER.info('[GetContextEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content)) # ----- Topology --------------------------------------------------------------------------------------------------- @@ -108,15 +146,18 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetTopology(self, request: Topology, context : grpc.ServicerContext) -> TopologyId: LOGGER.info('[SetTopology] request={:s}'.format(grpc_message_to_json_string(request))) container_name = 'topology[{:s}]'.format(str(request.topology_id.context_id.context_uuid.uuid)) - return set_entry(self.database, container_name, request.topology_id.topology_uuid.uuid, request).topology_id + topology_uuid = request.topology_id.topology_uuid.uuid + return self._set(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY) def RemoveTopology(self, request: TopologyId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveTopology] request={:s}'.format(grpc_message_to_json_string(request))) container_name = 'topology[{:s}]'.format(str(request.context_id.context_uuid.uuid)) - return del_entry(context, self.database, container_name, request.topology_uuid.uuid) + topology_uuid = request.topology_uuid.uuid + return self._del(request, container_name, topology_uuid, 'topology_id', TOPIC_TOPOLOGY, context) def GetTopologyEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[TopologyEvent]: LOGGER.info('[GetTopologyEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_TOPOLOGY}): yield TopologyEvent(**json.loads(message.content)) # ----- Device ----------------------------------------------------------------------------------------------------- @@ -135,14 +176,15 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetDevice(self, request: Context, context : grpc.ServicerContext) -> DeviceId: LOGGER.info('[SetDevice] request={:s}'.format(grpc_message_to_json_string(request))) - return set_entry(self.database, 'device', request.device_id.device_uuid.uuid, request).device_id + return self._set(request, 'device', request.device_id.device_uuid.uuid, 'device_id', TOPIC_DEVICE) def RemoveDevice(self, request: DeviceId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveDevice] request={:s}'.format(grpc_message_to_json_string(request))) - return del_entry(context, self.database, 'device', request.device_uuid.uuid) + return self._del(request, 'device', request.device_uuid.uuid, 'device_id', TOPIC_DEVICE, context) def GetDeviceEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[DeviceEvent]: LOGGER.info('[GetDeviceEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_DEVICE}): yield DeviceEvent(**json.loads(message.content)) # ----- Link ------------------------------------------------------------------------------------------------------- @@ -161,14 +203,15 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetLink(self, request: Context, context : grpc.ServicerContext) -> LinkId: LOGGER.info('[SetLink] request={:s}'.format(grpc_message_to_json_string(request))) - return set_entry(self.database, 'link', request.link_id.link_uuid.uuid, request).link_id + return self._set(request, 'link', request.link_id.link_uuid.uuid, 'link_id', TOPIC_LINK) def RemoveLink(self, request: LinkId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveLink] request={:s}'.format(grpc_message_to_json_string(request))) - return del_entry(context, self.database, 'link', request.link_uuid.uuid) + return self._del(request, 'link', request.link_uuid.uuid, 'link_id', TOPIC_LINK, context) def GetLinkEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[LinkEvent]: LOGGER.info('[GetLinkEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_LINK}): yield LinkEvent(**json.loads(message.content)) # ----- Slice ------------------------------------------------------------------------------------------------------ @@ -222,17 +265,19 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetService(self, request: Service, context : grpc.ServicerContext) -> ServiceId: LOGGER.info('[SetService] request={:s}'.format(grpc_message_to_json_string(request))) - return set_entry( - self.database, 'service[{:s}]'.format(str(request.service_id.context_id.context_uuid.uuid)), - request.service_id.service_uuid.uuid, request).service_id + container_name = 'service[{:s}]'.format(str(request.service_id.context_id.context_uuid.uuid)) + service_uuid = request.service_id.service_uuid.uuid + return self._set(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE) def RemoveService(self, request: ServiceId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveService] request={:s}'.format(grpc_message_to_json_string(request))) container_name = 'service[{:s}]'.format(str(request.context_id.context_uuid.uuid)) - return del_entry(context, self.database, container_name, request.service_uuid.uuid) + service_uuid = request.service_id.service_uuid.uuid + return self._del(request, container_name, service_uuid, 'service_id', TOPIC_SERVICE, context) def GetServiceEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ServiceEvent]: LOGGER.info('[GetServiceEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_SERVICE}): yield ServiceEvent(**json.loads(message.content)) # ----- Connection ------------------------------------------------------------------------------------------------- @@ -255,21 +300,21 @@ class MockServicerImpl_Context(ContextServiceServicer): def SetConnection(self, request: Connection, context : grpc.ServicerContext) -> ConnectionId: LOGGER.info('[SetConnection] request={:s}'.format(grpc_message_to_json_string(request))) - service_connection__container_name = 'service_connection[{:s}/{:s}]'.format( + container_name = 'service_connection[{:s}/{:s}]'.format( str(request.service_id.context_id.context_uuid.uuid), str(request.service_id.service_uuid.uuid)) - set_entry( - self.database, service_connection__container_name, request.connection_id.connection_uuid.uuid, request) - return set_entry( - self.database, 'connection', request.connection_id.connection_uuid.uuid, request).connection_id + connection_uuid = request.connection_id.connection_uuid.uuid + set_entry(self.database, container_name, connection_uuid, request) + return self._set(request, 'connection', connection_uuid, 'connection_id', TOPIC_CONNECTION) def RemoveConnection(self, request: ConnectionId, context : grpc.ServicerContext) -> Empty: LOGGER.info('[RemoveConnection] request={:s}'.format(grpc_message_to_json_string(request))) connection = get_entry(context, self.database, 'connection', request.connection_uuid.uuid) - service_id = connection.service_id - service_connection__container_name = 'service_connection[{:s}/{:s}]'.format( - str(service_id.context_id.context_uuid.uuid), str(service_id.service_uuid.uuid)) - del_entry(context, self.database, service_connection__container_name, request.connection_uuid.uuid) - return del_entry(context, self.database, 'connection', request.connection_uuid.uuid) + container_name = 'service_connection[{:s}/{:s}]'.format( + str(connection.service_id.context_id.context_uuid.uuid), str(connection.service_id.service_uuid.uuid)) + connection_uuid = request.connection_uuid.uuid + del_entry(context, self.database, container_name, connection_uuid) + return self._del(request, 'connection', connection_uuid, 'connection_id', TOPIC_CONNECTION, context) def GetConnectionEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ConnectionEvent]: LOGGER.info('[GetConnectionEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_CONNECTION}): yield ConnectionEvent(**json.loads(message.content)) diff --git a/src/common/tests/MockServicerImpl_Service copy.py b/src/common/tests/MockServicerImpl_Service copy.py new file mode 100644 index 000000000..3b5c769dd --- /dev/null +++ b/src/common/tests/MockServicerImpl_Service copy.py @@ -0,0 +1,108 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Iterator, NamedTuple, Tuple +import grpc, logging +from common.tests.MockMessageBroker import MockMessageBroker +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.proto.context_pb2 import Empty, TeraFlowController +from dlt.connector.proto.dlt_pb2 import ( + DltPeerStatus, DltPeerStatusList, DltRecord, DltRecordEvent, DltRecordId, DltRecordOperationEnum, DltRecordStatus, DltRecordSubscription, DltRecordTypeEnum) +from dlt.connector.proto.dlt_pb2_grpc import DltServiceServicer + +LOGGER = logging.getLogger(__name__) + +DltRecordKey = Tuple[str, DltRecordOperationEnum, str] # domain_uuid, operation, record_uuid +DltRecordDict = Dict[DltRecordKey, DltRecord] # dlt_record_key => dlt_record + +class MockServicerImpl_Dlt(DltServiceServicer): + def __init__(self): + LOGGER.info('[__init__] Creating Servicer...') + self.records : DltRecordDict = {} + self.msg_broker = MockMessageBroker() + LOGGER.info('[__init__] Servicer Created') + + def RecordToDlt(self, request : DltRecord, context : grpc.ServicerContext) -> DltRecordStatus: + LOGGER.info('[RecordToDlt] request={:s}'.format(grpc_message_to_json_string(request))) + operation = request.operation + domain_uuid = request.record_id.domain_uuid + record_uuid = request.record_id.record_uuid + + #if operation == + + + def GetFromDlt(self, request : DltRecordId, context : grpc.ServicerContext) -> DltRecord: + LOGGER.info('[GetFromDlt] request={:s}'.format(grpc_message_to_json_string(request))) + + def SubscribeToDlt(self, request: DltRecordSubscription, context : grpc.ServicerContext) -> Iterator[DltRecordEvent]: + LOGGER.info('[SubscribeToDlt] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content)) + + def GetDltStatus(self, request : TeraFlowController, context : grpc.ServicerContext) -> DltPeerStatus: + LOGGER.info('[GetDltStatus] request={:s}'.format(grpc_message_to_json_string(request))) + + def GetDltPeers(self, request : Empty, context : grpc.ServicerContext) -> DltPeerStatusList: + LOGGER.info('[GetDltPeers] request={:s}'.format(grpc_message_to_json_string(request))) + + + + + + + + LOGGER.info('[__init__] Servicer Created') + + # ----- Common ----------------------------------------------------------------------------------------------------- + + def _set(self, request, container_name, entry_uuid, entry_id_field_name, topic_name): + exists = has_entry(self.database, container_name, entry_uuid) + entry = set_entry(self.database, container_name, entry_uuid, request) + event_type = EventTypeEnum.EVENTTYPE_UPDATE if exists else EventTypeEnum.EVENTTYPE_CREATE + entry_id = getattr(entry, entry_id_field_name) + dict_entry_id = grpc_message_to_json(entry_id) + notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) + return entry_id + + def _del(self, request, container_name, entry_uuid, entry_id_field_name, topic_name, grpc_context): + empty = del_entry(grpc_context, self.database, container_name, entry_uuid) + event_type = EventTypeEnum.EVENTTYPE_REMOVE + dict_entry_id = grpc_message_to_json(request) + notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) + return empty + + # ----- Context ---------------------------------------------------------------------------------------------------- + + def ListContextIds(self, request: Empty, context : grpc.ServicerContext) -> ContextIdList: + LOGGER.info('[ListContextIds] request={:s}'.format(grpc_message_to_json_string(request))) + return ContextIdList(context_ids=[context.context_id for context in get_entries(self.database, 'context')]) + + def ListContexts(self, request: Empty, context : grpc.ServicerContext) -> ContextList: + LOGGER.info('[ListContexts] request={:s}'.format(grpc_message_to_json_string(request))) + return ContextList(contexts=get_entries(self.database, 'context')) + + def GetContext(self, request: ContextId, context : grpc.ServicerContext) -> Context: + LOGGER.info('[GetContext] request={:s}'.format(grpc_message_to_json_string(request))) + return get_entry(context, self.database, 'context', request.context_uuid.uuid) + + def SetContext(self, request: Context, context : grpc.ServicerContext) -> ContextId: + LOGGER.info('[SetContext] request={:s}'.format(grpc_message_to_json_string(request))) + return self._set(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT) + + def RemoveContext(self, request: ContextId, context : grpc.ServicerContext) -> Empty: + LOGGER.info('[RemoveContext] request={:s}'.format(grpc_message_to_json_string(request))) + return self._del(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT, context) + + def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]: + LOGGER.info('[GetContextEvents] request={:s}'.format(grpc_message_to_json_string(request))) + for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content)) diff --git a/src/dlt/connector/.gitlab-ci.yml b/src/dlt/connector/.gitlab-ci.yml new file mode 100644 index 000000000..d62e8edad --- /dev/null +++ b/src/dlt/connector/.gitlab-ci.yml @@ -0,0 +1,74 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker images to the GitLab Docker registry +build slice: + variables: + IMAGE_NAME: 'slice' # name of the microservice + IMAGE_NAME_TEST: 'slice-test' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + rules: + - changes: + - src/$IMAGE_NAME/** + - .gitlab-ci.yml + +# Pull, execute, and run unitary tests for the Docker image from the GitLab registry +unit_test slice: + variables: + IMAGE_NAME: 'slice' # name of the microservice + IMAGE_NAME_TEST: 'slice-test' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build slice + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run -d -p 4040:4040 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG" + - docker ps -a + - sleep 5 + - docker ps -a + - docker logs $IMAGE_NAME + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py" + after_script: + - docker stop $IMAGE_NAME + - docker rm $IMAGE_NAME + rules: + - changes: + - src/$IMAGE_NAME/** + - .gitlab-ci.yml + +# Deployment of the service in Kubernetes Cluster +deploy slice: + stage: deploy + needs: + - build slice + - unit_test slice + - dependencies all + - integ_test execute + script: + - kubectl version + - kubectl get all + - kubectl apply -f "manifests/sliceservice.yaml" + - kubectl delete pods --selector app=sliceservice + - kubectl get all diff --git a/src/dlt/connector/Dockerfile b/src/dlt/connector/Dockerfile new file mode 100644 index 000000000..d653bb217 --- /dev/null +++ b/src/dlt/connector/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools + +# Set working directory +WORKDIR /var/teraflow + +# Create module sub-folders +RUN mkdir -p /var/teraflow/slice + +# Get Python packages per module +COPY slice/requirements.in slice/requirements.in +RUN pip-compile --output-file=slice/requirements.txt slice/requirements.in +RUN python3 -m pip install -r slice/requirements.in + +# Add files into working directory +COPY common/. common +COPY context/. context +COPY interdomain/. interdomain +COPY service/. service +COPY slice/. slice + +# Start slice service +ENTRYPOINT ["python", "-m", "slice.service"] diff --git a/src/dlt/connector/client/DltClient.py b/src/dlt/connector/client/DltClient.py new file mode 100644 index 000000000..79d38d12d --- /dev/null +++ b/src/dlt/connector/client/DltClient.py @@ -0,0 +1,171 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import retry, delay_exponential +#from common.tools.grpc.Tools import grpc_message_to_json_string +#from slice.proto.context_pb2 import Empty, Slice, SliceId +from dlt.connector.proto.dlt_pb2_grpc import DltServiceStub + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class DltClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.DLT) + if not port: port = get_service_port_grpc(ServiceNameEnum.DLT) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = DltServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + +# @RETRY_DECORATOR +# def CreateSlice(self, request : Slice) -> SliceId: +# LOGGER.debug('CreateSlice request: {:s}'.format(grpc_message_to_json_string(request))) +# response = self.stub.CreateSlice(request) +# LOGGER.debug('CreateSlice result: {:s}'.format(grpc_message_to_json_string(response))) +# return response + +# @RETRY_DECORATOR +# def UpdateSlice(self, request : Slice) -> SliceId: +# LOGGER.debug('UpdateSlice request: {:s}'.format(grpc_message_to_json_string(request))) +# response = self.stub.UpdateSlice(request) +# LOGGER.debug('UpdateSlice result: {:s}'.format(grpc_message_to_json_string(response))) +# return response + +# @RETRY_DECORATOR +# def DeleteSlice(self, request : SliceId) -> Empty: +# LOGGER.debug('DeleteSlice request: {:s}'.format(grpc_message_to_json_string(request))) +# response = self.stub.DeleteSlice(request) +# LOGGER.debug('DeleteSlice result: {:s}'.format(grpc_message_to_json_string(response))) +# return response + + + + + + + +// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package dlt; + +import "context.proto"; + +service DltService { + rpc RecordToDlt (DltRecord ) returns ( DltRecordStatus ) {} + rpc GetFromDlt (DltRecordId ) returns ( DltRecord ) {} + rpc SubscribeToDlt(DltRecordSubscription ) returns (stream DltRecordEvent ) {} + rpc GetDltStatus (context.TeraFlowController) returns ( DltPeerStatus ) {} // NEC is checkig if it is possible + rpc GetDltPeers (context.Empty ) returns ( DltPeerStatusList) {} // NEC is checkig if it is possible +} + +enum DltRecordTypeEnum { + DLTRECORDTYPE_UNDEFINED = 0; + DLTRECORDTYPE_CONTEXT = 1; + DLTRECORDTYPE_TOPOLOGY = 2; + DLTRECORDTYPE_DEVICE = 3; + DLTRECORDTYPE_LINK = 4; + DLTRECORDTYPE_SERVICE = 5; + DLTRECORDTYPE_SLICE = 6; +} + +enum DltRecordOperationEnum { + DLTRECORDOPERATION_UNDEFINED = 0; + DLTRECORDOPERATION_ADD = 1; + DLTRECORDOPERATION_UPDATE = 2; + DLTRECORDOPERATION_DELETE = 3; +} + +enum DltRecordStatusEnum { + DLTRECORDSTATUS_UNDEFINED = 0; + DLTRECORDSTATUS_SUCCEEDED = 1; + DLTRECORDSTATUS_FAILED = 2; +} + +enum DltStatusEnum { + DLTSTATUS_UNDEFINED = 0; + DLTSTATUS_NOTAVAILABLE = 1; + DLTSTATUS_INITIALIZED = 2; + DLTSTATUS_AVAILABLE = 3; + DLTSTATUS_DEINIT = 4; +} + +message DltRecordId { + context.Uuid domain_uuid = 1; // unique identifier of domain owning the record + DltRecordTypeEnum type = 2; // type of record + context.Uuid record_uuid = 3; // unique identifier of the record within the domain context_uuid/topology_uuid +} + +message DltRecord { + DltRecordId record_id = 1; // record identifier + DltRecordOperationEnum operation = 2; // operation to be performed over the record + string data_json = 3; // record content: JSON-encoded record content +} + +message DltRecordSubscription { + // retrieved events have to match ALL conditions. + // i.e., type in types requested, AND operation in operations requested + // TODO: consider adding a more sophisticated filtering + repeated DltRecordTypeEnum type = 1; // selected event types, empty=all + repeated DltRecordOperationEnum operation = 2; // selected event operations, empty=all +} + +message DltRecordEvent { + context.Event event = 1; // common event data (timestamp & event_type) + DltRecordId record_id = 2; // record identifier associated with this event +} + +message DltRecordStatus { + DltRecordId record_id = 1; // identifier of the associated record + DltRecordStatusEnum status = 2; // status of the record + string error_message = 3; // error message in case of failure, empty otherwise +} + +message DltPeerStatus { + context.TeraFlowController controller = 1; // Identifier of the TeraFlow controller instance + DltStatusEnum status = 2; // Status of the TeraFlow controller instance +} + +message DltPeerStatusList { + repeated DltPeerStatus peers = 1; // List of peers and their status +} diff --git a/src/dlt/connector/service/DltConnector.py b/src/dlt/connector/service/DltConnector.py new file mode 100644 index 000000000..b8f7e5311 --- /dev/null +++ b/src/dlt/connector/service/DltConnector.py @@ -0,0 +1,51 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, threading +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient +from context.client.EventsCollector import EventsCollector +from dlt.connector.client.DltClient import DltClient + +LOGGER = logging.getLogger(__name__) + +class DltConnector: + def __init__(self) -> None: + LOGGER.debug('Creating connector...') + self._terminate = threading.Event() + self._thread = None + LOGGER.debug('Connector created') + + def start(self): + self._terminate.clear() + self._thread = threading.Thread(target=self._run_events_collector) + self._thread.start() + + def _run_events_collector(self) -> None: + dlt_client = DltClient() + context_client = ContextClient() + events_collector = EventsCollector(context_client) + events_collector.start() + + while not self._terminate.is_set(): + event = events_collector.get_event() + LOGGER.info('Event from Context Received: {:s}'.format(grpc_message_to_json_string(event))) + + events_collector.stop() + context_client.close() + dlt_client.close() + + def stop(self): + self._terminate.set() + self._thread.join() diff --git a/src/dlt/connector/service/DltConnectorService.py b/src/dlt/connector/service/DltConnectorService.py new file mode 100644 index 000000000..de696e88c --- /dev/null +++ b/src/dlt/connector/service/DltConnectorService.py @@ -0,0 +1,28 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from ..proto.dlt_pb2_grpc import add_SliceServiceServicer_to_server +from .DltConnectorServiceServicerImpl import SliceServiceServicerImpl + +class DltConnectorService(GenericGrpcService): + def __init__(self, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.SLICE) + super().__init__(port, cls_name=cls_name) + self.slice_servicer = SliceServiceServicerImpl() + + def install_servicers(self): + add_SliceServiceServicer_to_server(self.slice_servicer, self.server) diff --git a/src/dlt/connector/service/DltConnectorServiceServicerImpl.py b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py new file mode 100644 index 000000000..a19c797ea --- /dev/null +++ b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py @@ -0,0 +1,33 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method +from context.proto.context_pb2 import Slice, SliceId +from ..proto.dlt_pb2_grpc import DltServiceServicer + +LOGGER = logging.getLogger(__name__) + +SERVICE_NAME = 'DltConnector' +METHOD_NAMES = ['CreateSlice', 'UpdateSlice', 'DeleteSlice'] +METRICS = create_metrics(SERVICE_NAME, METHOD_NAMES) + +class DltConnectorServiceServicerImpl(DltServiceServicer): + def __init__(self): + LOGGER.debug('Creating Servicer...') + LOGGER.debug('Servicer Created') + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: + return SliceId() diff --git a/src/dlt/connector/service/__main__.py b/src/dlt/connector/service/__main__.py new file mode 100644 index 000000000..435a93f61 --- /dev/null +++ b/src/dlt/connector/service/__main__.py @@ -0,0 +1,65 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from prometheus_client import start_http_server +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, + wait_for_environment_variables) +from .DltConnectorService import DltConnectorService + +terminate = threading.Event() +LOGGER : logging.Logger = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting DLT connector service + grpc_service = DltConnectorService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=0.1): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/dlt/connector/tests/test_unitary.py b/src/dlt/connector/tests/test_unitary.py new file mode 100644 index 000000000..0f1fcb3eb --- /dev/null +++ b/src/dlt/connector/tests/test_unitary.py @@ -0,0 +1,49 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Tuple +from common.orm.Database import Database +from common.message_broker.MessageBroker import MessageBroker +from context.client.ContextClient import ContextClient +from context.client.EventsCollector import EventsCollector +from context.proto.context_pb2 import Context, ContextId, Device, DeviceId, Link, LinkId, Topology, TopologyId +from .PrepareTestScenario import context_db_mb, context_service, context_client # pylint: disable=unused-import +from .Objects import CONTEXTS, TOPOLOGIES, DEVICES, LINKS + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +def test_create_events( + context_client : ContextClient, # pylint: disable=redefined-outer-name + context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name + context_database = context_db_mb[0] + + context_database.clear_all() + + events_collector = EventsCollector(context_client) + events_collector.start() + events = events_collector.get_events(block=True, count=4) + events_collector.stop() + + for context in CONTEXTS : context_client.SetContext (Context (**context )) + for topology in TOPOLOGIES: context_client.SetTopology(Topology(**topology)) + for device in DEVICES : context_client.SetDevice (Device (**device )) + for link in LINKS : context_client.SetLink (Link (**link )) + + + for link in LINKS : context_client.RemoveLink (LinkId (**link ['link_id' ])) + for device in DEVICES : context_client.RemoveDevice (DeviceId (**device ['device_id' ])) + for topology in TOPOLOGIES: context_client.RemoveTopology(TopologyId(**topology['topology_id'])) + for context in CONTEXTS : context_client.RemoveContext (ContextId (**context ['context_id' ])) -- GitLab From 28cceac3924a9689a2fce0a9e2be5437b8022b4b Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Mon, 18 Jul 2022 16:53:00 +0200 Subject: [PATCH 05/36] DLTConnector component: - Implemented DLTConnectorClient and DLTGatewayClient - Implemented DLTConnectorService skeleton --- src/dlt/connector/Config.py | 1 - src/dlt/connector/client/DltClient.py | 171 ------------------ .../connector/client/DltConnectorClient.py | 95 ++++++++++ src/dlt/connector/client/DltGatewayClient.py | 84 +++++++++ src/dlt/connector/client/__init__.py | 1 - src/dlt/connector/service/DltConnector.py | 6 +- .../connector/service/DltConnectorService.py | 10 +- .../DltConnectorServiceServicerImpl.py | 41 ++++- src/dlt/connector/service/__init__.py | 1 - 9 files changed, 222 insertions(+), 188 deletions(-) delete mode 100644 src/dlt/connector/client/DltClient.py create mode 100644 src/dlt/connector/client/DltConnectorClient.py create mode 100644 src/dlt/connector/client/DltGatewayClient.py diff --git a/src/dlt/connector/Config.py b/src/dlt/connector/Config.py index 70a332512..9953c8205 100644 --- a/src/dlt/connector/Config.py +++ b/src/dlt/connector/Config.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/src/dlt/connector/client/DltClient.py b/src/dlt/connector/client/DltClient.py deleted file mode 100644 index 79d38d12d..000000000 --- a/src/dlt/connector/client/DltClient.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import grpc, logging -from common.Constants import ServiceNameEnum -from common.Settings import get_service_host, get_service_port_grpc -from common.tools.client.RetryDecorator import retry, delay_exponential -#from common.tools.grpc.Tools import grpc_message_to_json_string -#from slice.proto.context_pb2 import Empty, Slice, SliceId -from dlt.connector.proto.dlt_pb2_grpc import DltServiceStub - -LOGGER = logging.getLogger(__name__) -MAX_RETRIES = 15 -DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) -RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') - -class DltClient: - def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.DLT) - if not port: port = get_service_port_grpc(ServiceNameEnum.DLT) - self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) - LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) - self.channel = None - self.stub = None - self.connect() - LOGGER.debug('Channel created') - - def connect(self): - self.channel = grpc.insecure_channel(self.endpoint) - self.stub = DltServiceStub(self.channel) - - def close(self): - if self.channel is not None: self.channel.close() - self.channel = None - self.stub = None - -# @RETRY_DECORATOR -# def CreateSlice(self, request : Slice) -> SliceId: -# LOGGER.debug('CreateSlice request: {:s}'.format(grpc_message_to_json_string(request))) -# response = self.stub.CreateSlice(request) -# LOGGER.debug('CreateSlice result: {:s}'.format(grpc_message_to_json_string(response))) -# return response - -# @RETRY_DECORATOR -# def UpdateSlice(self, request : Slice) -> SliceId: -# LOGGER.debug('UpdateSlice request: {:s}'.format(grpc_message_to_json_string(request))) -# response = self.stub.UpdateSlice(request) -# LOGGER.debug('UpdateSlice result: {:s}'.format(grpc_message_to_json_string(response))) -# return response - -# @RETRY_DECORATOR -# def DeleteSlice(self, request : SliceId) -> Empty: -# LOGGER.debug('DeleteSlice request: {:s}'.format(grpc_message_to_json_string(request))) -# response = self.stub.DeleteSlice(request) -# LOGGER.debug('DeleteSlice result: {:s}'.format(grpc_message_to_json_string(response))) -# return response - - - - - - - -// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package dlt; - -import "context.proto"; - -service DltService { - rpc RecordToDlt (DltRecord ) returns ( DltRecordStatus ) {} - rpc GetFromDlt (DltRecordId ) returns ( DltRecord ) {} - rpc SubscribeToDlt(DltRecordSubscription ) returns (stream DltRecordEvent ) {} - rpc GetDltStatus (context.TeraFlowController) returns ( DltPeerStatus ) {} // NEC is checkig if it is possible - rpc GetDltPeers (context.Empty ) returns ( DltPeerStatusList) {} // NEC is checkig if it is possible -} - -enum DltRecordTypeEnum { - DLTRECORDTYPE_UNDEFINED = 0; - DLTRECORDTYPE_CONTEXT = 1; - DLTRECORDTYPE_TOPOLOGY = 2; - DLTRECORDTYPE_DEVICE = 3; - DLTRECORDTYPE_LINK = 4; - DLTRECORDTYPE_SERVICE = 5; - DLTRECORDTYPE_SLICE = 6; -} - -enum DltRecordOperationEnum { - DLTRECORDOPERATION_UNDEFINED = 0; - DLTRECORDOPERATION_ADD = 1; - DLTRECORDOPERATION_UPDATE = 2; - DLTRECORDOPERATION_DELETE = 3; -} - -enum DltRecordStatusEnum { - DLTRECORDSTATUS_UNDEFINED = 0; - DLTRECORDSTATUS_SUCCEEDED = 1; - DLTRECORDSTATUS_FAILED = 2; -} - -enum DltStatusEnum { - DLTSTATUS_UNDEFINED = 0; - DLTSTATUS_NOTAVAILABLE = 1; - DLTSTATUS_INITIALIZED = 2; - DLTSTATUS_AVAILABLE = 3; - DLTSTATUS_DEINIT = 4; -} - -message DltRecordId { - context.Uuid domain_uuid = 1; // unique identifier of domain owning the record - DltRecordTypeEnum type = 2; // type of record - context.Uuid record_uuid = 3; // unique identifier of the record within the domain context_uuid/topology_uuid -} - -message DltRecord { - DltRecordId record_id = 1; // record identifier - DltRecordOperationEnum operation = 2; // operation to be performed over the record - string data_json = 3; // record content: JSON-encoded record content -} - -message DltRecordSubscription { - // retrieved events have to match ALL conditions. - // i.e., type in types requested, AND operation in operations requested - // TODO: consider adding a more sophisticated filtering - repeated DltRecordTypeEnum type = 1; // selected event types, empty=all - repeated DltRecordOperationEnum operation = 2; // selected event operations, empty=all -} - -message DltRecordEvent { - context.Event event = 1; // common event data (timestamp & event_type) - DltRecordId record_id = 2; // record identifier associated with this event -} - -message DltRecordStatus { - DltRecordId record_id = 1; // identifier of the associated record - DltRecordStatusEnum status = 2; // status of the record - string error_message = 3; // error message in case of failure, empty otherwise -} - -message DltPeerStatus { - context.TeraFlowController controller = 1; // Identifier of the TeraFlow controller instance - DltStatusEnum status = 2; // Status of the TeraFlow controller instance -} - -message DltPeerStatusList { - repeated DltPeerStatus peers = 1; // List of peers and their status -} diff --git a/src/dlt/connector/client/DltConnectorClient.py b/src/dlt/connector/client/DltConnectorClient.py new file mode 100644 index 000000000..f48562996 --- /dev/null +++ b/src/dlt/connector/client/DltConnectorClient.py @@ -0,0 +1,95 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.proto.context_pb2 import DeviceId, Empty, ServiceId, SliceId +from common.proto.dlt_connector_pb2_grpc import DltConnectorServiceStub +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class DltConnectorClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.DLT) + if not port: port = get_service_port_grpc(ServiceNameEnum.DLT) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = DltConnectorServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def RecordAll(self, request : Empty) -> Empty: + LOGGER.debug('RecordAll request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordAll(request) + LOGGER.debug('RecordAll result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordAllDevices(self, request : Empty) -> Empty: + LOGGER.debug('RecordAllDevices request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordAllDevices(request) + LOGGER.debug('RecordAllDevices result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordDevice(self, request : DeviceId) -> Empty: + LOGGER.debug('RecordDevice request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordDevice(request) + LOGGER.debug('RecordDevice result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordAllServices(self, request : Empty) -> Empty: + LOGGER.debug('RecordAllServices request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordAllServices(request) + LOGGER.debug('RecordAllServices result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordService(self, request : ServiceId) -> Empty: + LOGGER.debug('RecordService request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordService(request) + LOGGER.debug('RecordService result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordAllSlices(self, request : Empty) -> Empty: + LOGGER.debug('RecordAllSlices request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordAllSlices(request) + LOGGER.debug('RecordAllSlices result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def RecordSlice(self, request : SliceId) -> Empty: + LOGGER.debug('RecordSlice request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordSlice(request) + LOGGER.debug('RecordSlice result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/dlt/connector/client/DltGatewayClient.py b/src/dlt/connector/client/DltGatewayClient.py new file mode 100644 index 000000000..f1f8dec39 --- /dev/null +++ b/src/dlt/connector/client/DltGatewayClient.py @@ -0,0 +1,84 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterator +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.proto.context_pb2 import Empty, TeraFlowController +from common.proto.dlt_gateway_pb2 import ( + DltPeerStatus, DltPeerStatusList, DltRecord, DltRecordEvent, DltRecordId, DltRecordStatus, DltRecordSubscription) +from common.proto.dlt_gateway_pb2_grpc import DltGatewayServiceStub +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class DltGatewayClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.DLT) + if not port: port = get_service_port_grpc(ServiceNameEnum.DLT) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = DltGatewayServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def RecordToDlt(self, request : DltRecord) -> DltRecordStatus: + LOGGER.debug('RecordToDlt request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.RecordToDlt(request) + LOGGER.debug('RecordToDlt result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetFromDlt(self, request : DltRecordId) -> DltRecord: + LOGGER.debug('GetFromDlt request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetFromDlt(request) + LOGGER.debug('GetFromDlt result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SubscribeToDlt(self, request : DltRecordSubscription) -> Iterator[DltRecordEvent]: + LOGGER.debug('SubscribeToDlt request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SubscribeToDlt(request) + LOGGER.debug('SubscribeToDlt result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetDltStatus(self, request : TeraFlowController) -> DltPeerStatus: + LOGGER.debug('GetDltStatus request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetDltStatus(request) + LOGGER.debug('GetDltStatus result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetDltPeers(self, request : Empty) -> DltPeerStatusList: + LOGGER.debug('GetDltPeers request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetDltPeers(request) + LOGGER.debug('GetDltPeers result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/dlt/connector/client/__init__.py b/src/dlt/connector/client/__init__.py index 70a332512..9953c8205 100644 --- a/src/dlt/connector/client/__init__.py +++ b/src/dlt/connector/client/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/src/dlt/connector/service/DltConnector.py b/src/dlt/connector/service/DltConnector.py index b8f7e5311..0c42d6685 100644 --- a/src/dlt/connector/service/DltConnector.py +++ b/src/dlt/connector/service/DltConnector.py @@ -16,7 +16,7 @@ import logging, threading from common.tools.grpc.Tools import grpc_message_to_json_string from context.client.ContextClient import ContextClient from context.client.EventsCollector import EventsCollector -from dlt.connector.client.DltClient import DltClient +from dlt.connector.client.DltConnectorClient import DltConnectorClient LOGGER = logging.getLogger(__name__) @@ -33,7 +33,7 @@ class DltConnector: self._thread.start() def _run_events_collector(self) -> None: - dlt_client = DltClient() + dltconnector_client = DltConnectorClient() context_client = ContextClient() events_collector = EventsCollector(context_client) events_collector.start() @@ -44,7 +44,7 @@ class DltConnector: events_collector.stop() context_client.close() - dlt_client.close() + dltconnector_client.close() def stop(self): self._terminate.set() diff --git a/src/dlt/connector/service/DltConnectorService.py b/src/dlt/connector/service/DltConnectorService.py index de696e88c..40237b628 100644 --- a/src/dlt/connector/service/DltConnectorService.py +++ b/src/dlt/connector/service/DltConnectorService.py @@ -15,14 +15,14 @@ from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc from common.tools.service.GenericGrpcService import GenericGrpcService -from ..proto.dlt_pb2_grpc import add_SliceServiceServicer_to_server -from .DltConnectorServiceServicerImpl import SliceServiceServicerImpl +from common.proto.dlt_connector_pb2_grpc import add_DltConnectorServiceServicer_to_server +from .DltConnectorServiceServicerImpl import DltConnectorServiceServicerImpl class DltConnectorService(GenericGrpcService): def __init__(self, cls_name: str = __name__) -> None: - port = get_service_port_grpc(ServiceNameEnum.SLICE) + port = get_service_port_grpc(ServiceNameEnum.DLT) super().__init__(port, cls_name=cls_name) - self.slice_servicer = SliceServiceServicerImpl() + self.dltconnector_servicer = DltConnectorServiceServicerImpl() def install_servicers(self): - add_SliceServiceServicer_to_server(self.slice_servicer, self.server) + add_DltConnectorServiceServicer_to_server(self.dltconnector_servicer, self.server) diff --git a/src/dlt/connector/service/DltConnectorServiceServicerImpl.py b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py index a19c797ea..860e46f3a 100644 --- a/src/dlt/connector/service/DltConnectorServiceServicerImpl.py +++ b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py @@ -14,20 +14,49 @@ import grpc, logging from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method -from context.proto.context_pb2 import Slice, SliceId -from ..proto.dlt_pb2_grpc import DltServiceServicer +from common.proto.context_pb2 import DeviceId, Empty, ServiceId, SliceId +from common.proto.dlt_connector_pb2_grpc import DltConnectorServiceServicer LOGGER = logging.getLogger(__name__) SERVICE_NAME = 'DltConnector' -METHOD_NAMES = ['CreateSlice', 'UpdateSlice', 'DeleteSlice'] +METHOD_NAMES = [ + 'RecordAll', + 'RecordAllDevices', 'RecordDevice', + 'RecordAllServices', 'RecordService', + 'RecordAllSlices', 'RecordSlice', +] METRICS = create_metrics(SERVICE_NAME, METHOD_NAMES) -class DltConnectorServiceServicerImpl(DltServiceServicer): +class DltConnectorServiceServicerImpl(DltConnectorServiceServicer): def __init__(self): LOGGER.debug('Creating Servicer...') LOGGER.debug('Servicer Created') @safe_and_metered_rpc_method(METRICS, LOGGER) - def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: - return SliceId() + def RecordAll(self, request : Empty, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordAllDevices(self, request : Empty, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordDevice(self, request : DeviceId, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordAllServices(self, request : Empty, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordService(self, request : ServiceId, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordAllSlices(self, request : Empty, context : grpc.ServicerContext) -> Empty: + return Empty() + + @safe_and_metered_rpc_method(METRICS, LOGGER) + def RecordSlice(self, request : SliceId, context : grpc.ServicerContext) -> Empty: + return Empty() diff --git a/src/dlt/connector/service/__init__.py b/src/dlt/connector/service/__init__.py index 70a332512..9953c8205 100644 --- a/src/dlt/connector/service/__init__.py +++ b/src/dlt/connector/service/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -- GitLab From 231574445c68da8c97daaed21d6892d2cf74aec9 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Mon, 18 Jul 2022 18:59:17 +0200 Subject: [PATCH 06/36] DltConnector component: - Implemented MockServicerImpl_DltGateway - Updated Objects for unitary tests --- .../tests/MockServicerImpl_DltGateway.py | 165 ++++++++++++++++++ .../tests/MockServicerImpl_Service copy.py | 108 ------------ src/dlt/connector/tests/Objects.py | 58 ++++-- src/dlt/connector/tests/__init__.py | 1 - 4 files changed, 208 insertions(+), 124 deletions(-) create mode 100644 src/common/tests/MockServicerImpl_DltGateway.py delete mode 100644 src/common/tests/MockServicerImpl_Service copy.py diff --git a/src/common/tests/MockServicerImpl_DltGateway.py b/src/common/tests/MockServicerImpl_DltGateway.py new file mode 100644 index 000000000..16eae7a34 --- /dev/null +++ b/src/common/tests/MockServicerImpl_DltGateway.py @@ -0,0 +1,165 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, itertools, json, logging, time +from typing import Dict, Iterator, Optional, Tuple +from common.tests.MockMessageBroker import Message, MockMessageBroker +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.proto.context_pb2 import EVENTTYPE_CREATE, EVENTTYPE_REMOVE, EVENTTYPE_UPDATE, Empty, TeraFlowController +from common.proto.dlt_gateway_pb2 import ( + DLTRECORDOPERATION_ADD, DLTRECORDOPERATION_DELETE, DLTRECORDOPERATION_UNDEFINED, DLTRECORDOPERATION_UPDATE, + DLTRECORDSTATUS_FAILED, DLTRECORDSTATUS_SUCCEEDED, DLTRECORDTYPE_CONTEXT, DLTRECORDTYPE_DEVICE, DLTRECORDTYPE_LINK, + DLTRECORDTYPE_SERVICE, DLTRECORDTYPE_SLICE, DLTRECORDTYPE_TOPOLOGY, DLTRECORDTYPE_UNDEFINED, + DltPeerStatus, DltPeerStatusList, DltRecord, DltRecordEvent, DltRecordId, DltRecordOperationEnum, DltRecordStatus, + DltRecordSubscription, DltRecordTypeEnum) +from common.proto.dlt_gateway_pb2_grpc import DltGatewayServiceServicer + +LOGGER = logging.getLogger(__name__) + +DltRecordKey = Tuple[str, DltRecordOperationEnum, str] # domain_uuid, operation, record_uuid +DltRecordDict = Dict[DltRecordKey, DltRecord] # dlt_record_key => dlt_record + +class AlreadyExistsException(Exception): + pass + +class DoesNotExistException(Exception): + pass + +class MockServicerImpl_DltGateway(DltGatewayServiceServicer): + def __init__(self): + LOGGER.info('[__init__] Creating Servicer...') + self.records : DltRecordDict = {} + self.msg_broker = MockMessageBroker() + LOGGER.info('[__init__] Servicer Created') + + def __get_record(self, record_id : DltRecordId, should_exist : bool) -> Optional[Dict]: + domain_uuid, record_uuid = record_id.domain_uuid.uuid, record_id.record_uuid.uuid + str_type = DltRecordTypeEnum.Name(record_id.type).upper().replace('DLTRECORDTYPE_', '') + records_domain : Dict[str, Dict] = self.records.setdefault(domain_uuid, {}) + records_type : Dict[str, Dict] = records_domain.setdefault(str_type, {}) + record : Optional[Dict] = records_type.get(record_uuid) + if should_exist and record is None: + raise DoesNotExistException('RecordId({:s}, {:s}, {:s})'.format(domain_uuid, str_type, record_uuid)) + elif not should_exist and record is not None: + raise AlreadyExistsException('RecordId({:s}, {:s}, {:s})'.format(domain_uuid, str_type, record_uuid)) + return record + + def __set_record(self, record_id : DltRecordId, should_exist : bool, data_json : str) -> None: + domain_uuid, record_uuid = record_id.domain_uuid.uuid, record_id.record_uuid.uuid + str_type = DltRecordTypeEnum.Name(record_id.type).upper().replace('DLTRECORDTYPE_', '') + records_domain : Dict[str, Dict] = self.records.setdefault(domain_uuid, {}) + records_type : Dict[str, Dict] = records_domain.setdefault(str_type, {}) + record : Optional[Dict] = records_type.get(record_uuid) + if should_exist and record is None: + raise DoesNotExistException('RecordId({:s}, {:s}, {:s})'.format(domain_uuid, str_type, record_uuid)) + elif not should_exist and record is not None: + raise AlreadyExistsException('RecordId({:s}, {:s}, {:s})'.format(domain_uuid, str_type, record_uuid)) + records_type[record_uuid] = json.loads(data_json) + + def __del_record(self, record_id : DltRecordId) -> None: + domain_uuid, record_uuid = record_id.domain_uuid.uuid, record_id.record_uuid.uuid + str_type = DltRecordTypeEnum.Name(record_id.type).upper().replace('DLTRECORDTYPE_', '') + records_domain : Dict[str, Dict] = self.records.setdefault(domain_uuid, {}) + records_type : Dict[str, Dict] = records_domain.setdefault(str_type, {}) + record : Optional[Dict] = records_type.get(record_uuid) + if record is None: + raise DoesNotExistException('RecordId({:s}, {:s}, {:s})'.format(domain_uuid, str_type, record_uuid)) + records_type.discard(record_uuid) + + def __publish(self, operation : DltRecordOperationEnum, record_id : DltRecordId) -> None: + str_operation = DltRecordOperationEnum.Name(operation).upper().replace('DLTRECORDOPERATION_', '') + str_type = DltRecordTypeEnum.Name(record_id.type).upper().replace('DLTRECORDTYPE_', '') + topic = '{:s}:{:s}'.format(str_type, str_operation) + event = DltRecordEvent() + event.event.timestamp.timestamp = time.time() # pylint: disable=no-member + event.event.event_type = { # pylint: disable=no-member + DLTRECORDOPERATION_ADD : EVENTTYPE_CREATE, + DLTRECORDOPERATION_UPDATE: EVENTTYPE_UPDATE, + DLTRECORDOPERATION_DELETE: EVENTTYPE_REMOVE, + }.get(operation) + event.record_id.CopyFrom(record_id) # pylint: disable=no-member + self.msg_broker.publish(Message(topic=topic, content=grpc_message_to_json_string(event))) + + def RecordToDlt(self, request : DltRecord, context : grpc.ServicerContext) -> DltRecordStatus: + LOGGER.info('[RecordToDlt] request={:s}'.format(grpc_message_to_json_string(request))) + record_id = request.record_id + response = DltRecordStatus() + response.record_id.CopyFrom(record_id) # pylint: disable=no-member + try: + operation : DltRecordOperationEnum = request.operation + if operation == DLTRECORDOPERATION_ADD: + self.__set_record(record_id, False, request.data_json) + elif operation == DLTRECORDOPERATION_UPDATE: + self.__set_record(record_id, True, request.data_json) + elif operation == DLTRECORDOPERATION_DELETE: + self.__del_record(record_id) + else: + str_operation = DltRecordOperationEnum.Name(operation).upper().replace('DLTRECORDOPERATION_', '') + raise NotImplementedError('DltRecordOperationEnum({:s})'.format(str_operation)) + self.__publish(operation, record_id) + response.status = DLTRECORDSTATUS_SUCCEEDED + except Exception as e: # pylint: disable=broad-except + response.status = DLTRECORDSTATUS_FAILED + response.error_message = str(e) + LOGGER.info('[RecordToDlt] response={:s}'.format(grpc_message_to_json_string(response))) + return response + + def GetFromDlt(self, request : DltRecordId, context : grpc.ServicerContext) -> DltRecord: + LOGGER.info('[GetFromDlt] request={:s}'.format(grpc_message_to_json_string(request))) + record = self.__get_record(request, True) + response = DltRecord() + response.record_id.CopyFrom(request) # pylint: disable=no-member + response.operation = DLTRECORDOPERATION_UNDEFINED + response.data_json = json.dumps(record, sort_keys=True) + LOGGER.info('[GetFromDlt] response={:s}'.format(grpc_message_to_json_string(response))) + return response + + def SubscribeToDlt( + self, request: DltRecordSubscription, context : grpc.ServicerContext + ) -> Iterator[DltRecordEvent]: + LOGGER.info('[SubscribeToDlt] request={:s}'.format(grpc_message_to_json_string(request))) + types = request.type + if len(types) == 0: + types = [ + DLTRECORDTYPE_UNDEFINED, DLTRECORDTYPE_CONTEXT, DLTRECORDTYPE_TOPOLOGY, DLTRECORDTYPE_DEVICE, + DLTRECORDTYPE_LINK, DLTRECORDTYPE_SERVICE, DLTRECORDTYPE_SLICE + ] + str_types = [ + DltRecordTypeEnum.Name(_type).upper().replace('DLTRECORDTYPE_', '') + for _type in types + ] + operations = request.operation + if len(operations) == 0: + operations = [ + DLTRECORDOPERATION_UNDEFINED, DLTRECORDOPERATION_ADD, DLTRECORDOPERATION_UPDATE, + DLTRECORDOPERATION_DELETE + ] + str_operations = [ + DltRecordOperationEnum.Name(_operation).upper().replace('DLTRECORDOPERATION_', '') + for _operation in operations + ] + topics = { + '{:s}:{:s}'.format(*type_operation) + for type_operation in itertools.product(str_types, str_operations) + } + for message in self.msg_broker.consume(topics): + yield DltRecordEvent(**json.loads(message.content)) + + def GetDltStatus(self, request : TeraFlowController, context : grpc.ServicerContext) -> DltPeerStatus: + LOGGER.info('[GetDltStatus] request={:s}'.format(grpc_message_to_json_string(request))) + raise NotImplementedError() + + def GetDltPeers(self, request : Empty, context : grpc.ServicerContext) -> DltPeerStatusList: + LOGGER.info('[GetDltPeers] request={:s}'.format(grpc_message_to_json_string(request))) + raise NotImplementedError() diff --git a/src/common/tests/MockServicerImpl_Service copy.py b/src/common/tests/MockServicerImpl_Service copy.py deleted file mode 100644 index 3b5c769dd..000000000 --- a/src/common/tests/MockServicerImpl_Service copy.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict, Iterator, NamedTuple, Tuple -import grpc, logging -from common.tests.MockMessageBroker import MockMessageBroker -from common.tools.grpc.Tools import grpc_message_to_json_string -from context.proto.context_pb2 import Empty, TeraFlowController -from dlt.connector.proto.dlt_pb2 import ( - DltPeerStatus, DltPeerStatusList, DltRecord, DltRecordEvent, DltRecordId, DltRecordOperationEnum, DltRecordStatus, DltRecordSubscription, DltRecordTypeEnum) -from dlt.connector.proto.dlt_pb2_grpc import DltServiceServicer - -LOGGER = logging.getLogger(__name__) - -DltRecordKey = Tuple[str, DltRecordOperationEnum, str] # domain_uuid, operation, record_uuid -DltRecordDict = Dict[DltRecordKey, DltRecord] # dlt_record_key => dlt_record - -class MockServicerImpl_Dlt(DltServiceServicer): - def __init__(self): - LOGGER.info('[__init__] Creating Servicer...') - self.records : DltRecordDict = {} - self.msg_broker = MockMessageBroker() - LOGGER.info('[__init__] Servicer Created') - - def RecordToDlt(self, request : DltRecord, context : grpc.ServicerContext) -> DltRecordStatus: - LOGGER.info('[RecordToDlt] request={:s}'.format(grpc_message_to_json_string(request))) - operation = request.operation - domain_uuid = request.record_id.domain_uuid - record_uuid = request.record_id.record_uuid - - #if operation == - - - def GetFromDlt(self, request : DltRecordId, context : grpc.ServicerContext) -> DltRecord: - LOGGER.info('[GetFromDlt] request={:s}'.format(grpc_message_to_json_string(request))) - - def SubscribeToDlt(self, request: DltRecordSubscription, context : grpc.ServicerContext) -> Iterator[DltRecordEvent]: - LOGGER.info('[SubscribeToDlt] request={:s}'.format(grpc_message_to_json_string(request))) - for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content)) - - def GetDltStatus(self, request : TeraFlowController, context : grpc.ServicerContext) -> DltPeerStatus: - LOGGER.info('[GetDltStatus] request={:s}'.format(grpc_message_to_json_string(request))) - - def GetDltPeers(self, request : Empty, context : grpc.ServicerContext) -> DltPeerStatusList: - LOGGER.info('[GetDltPeers] request={:s}'.format(grpc_message_to_json_string(request))) - - - - - - - - LOGGER.info('[__init__] Servicer Created') - - # ----- Common ----------------------------------------------------------------------------------------------------- - - def _set(self, request, container_name, entry_uuid, entry_id_field_name, topic_name): - exists = has_entry(self.database, container_name, entry_uuid) - entry = set_entry(self.database, container_name, entry_uuid, request) - event_type = EventTypeEnum.EVENTTYPE_UPDATE if exists else EventTypeEnum.EVENTTYPE_CREATE - entry_id = getattr(entry, entry_id_field_name) - dict_entry_id = grpc_message_to_json(entry_id) - notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) - return entry_id - - def _del(self, request, container_name, entry_uuid, entry_id_field_name, topic_name, grpc_context): - empty = del_entry(grpc_context, self.database, container_name, entry_uuid) - event_type = EventTypeEnum.EVENTTYPE_REMOVE - dict_entry_id = grpc_message_to_json(request) - notify_event(self.msg_broker, topic_name, event_type, {entry_id_field_name: dict_entry_id}) - return empty - - # ----- Context ---------------------------------------------------------------------------------------------------- - - def ListContextIds(self, request: Empty, context : grpc.ServicerContext) -> ContextIdList: - LOGGER.info('[ListContextIds] request={:s}'.format(grpc_message_to_json_string(request))) - return ContextIdList(context_ids=[context.context_id for context in get_entries(self.database, 'context')]) - - def ListContexts(self, request: Empty, context : grpc.ServicerContext) -> ContextList: - LOGGER.info('[ListContexts] request={:s}'.format(grpc_message_to_json_string(request))) - return ContextList(contexts=get_entries(self.database, 'context')) - - def GetContext(self, request: ContextId, context : grpc.ServicerContext) -> Context: - LOGGER.info('[GetContext] request={:s}'.format(grpc_message_to_json_string(request))) - return get_entry(context, self.database, 'context', request.context_uuid.uuid) - - def SetContext(self, request: Context, context : grpc.ServicerContext) -> ContextId: - LOGGER.info('[SetContext] request={:s}'.format(grpc_message_to_json_string(request))) - return self._set(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT) - - def RemoveContext(self, request: ContextId, context : grpc.ServicerContext) -> Empty: - LOGGER.info('[RemoveContext] request={:s}'.format(grpc_message_to_json_string(request))) - return self._del(request, 'context', request.context_uuid.uuid, 'context_id', TOPIC_CONTEXT, context) - - def GetContextEvents(self, request: Empty, context : grpc.ServicerContext) -> Iterator[ContextEvent]: - LOGGER.info('[GetContextEvents] request={:s}'.format(grpc_message_to_json_string(request))) - for message in self.msg_broker.consume({TOPIC_CONTEXT}): yield ContextEvent(**json.loads(message.content)) diff --git a/src/dlt/connector/tests/Objects.py b/src/dlt/connector/tests/Objects.py index cb3baf9c9..f797e93e6 100644 --- a/src/dlt/connector/tests/Objects.py +++ b/src/dlt/connector/tests/Objects.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from common.Constants import DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID from common.tools.object_factory.Context import json_context, json_context_id from common.tools.object_factory.Device import json_device_emulated_packet_router_disabled, json_device_id from common.tools.object_factory.EndPoint import json_endpoints @@ -28,26 +27,55 @@ def compose_device( device = json_device_emulated_packet_router_disabled(device_uuid, endpoints=endpoints) return device_id, endpoints, device +# ===== Domain A ======================================================================================================= + +# ----- Context -------------------------------------------------------------------------------------------------------- +DA_CONTEXT_ADMIN_ID = json_context_id('A') +DA_CONTEXT_ADMIN = json_context('A') + +# ----- Topology ------------------------------------------------------------------------------------------------------- +DA_TOPOLOGY_ADMIN_ID = json_topology_id('A', context_id=DA_CONTEXT_ADMIN_ID) +DA_TOPOLOGY_ADMIN = json_topology('A', context_id=DA_CONTEXT_ADMIN_ID) + +# ----- Devices -------------------------------------------------------------------------------------------------------- +DA_DEVICE_DEV1_ID, DA_DEVICE_DEV1_ENDPOINTS, DA_DEVICE_DEV1 = compose_device('DEV1@A', ['1', '2']) +DA_DEVICE_DEV2_ID, DA_DEVICE_DEV2_ENDPOINTS, DA_DEVICE_DEV2 = compose_device('DEV2@A', ['1', '2']) +DA_DEVICE_DEV3_ID, DA_DEVICE_DEV3_ENDPOINTS, DA_DEVICE_DEV3 = compose_device('DEV3@A', ['1', '2']) + +# ----- Links ---------------------------------------------------------------------------------------------------------- +DA_LINK_DEV1_DEV2_ID, DA_LINK_DEV1_DEV2 = compose_link(DA_DEVICE_DEV1_ENDPOINTS[0], DA_DEVICE_DEV2_ENDPOINTS[0]) +DA_LINK_DEV1_DEV3_ID, DA_LINK_DEV1_DEV3 = compose_link(DA_DEVICE_DEV1_ENDPOINTS[1], DA_DEVICE_DEV3_ENDPOINTS[0]) +DA_LINK_DEV2_DEV3_ID, DA_LINK_DEV2_DEV3 = compose_link(DA_DEVICE_DEV2_ENDPOINTS[1], DA_DEVICE_DEV3_ENDPOINTS[1]) + +# ----- Containers ----------------------------------------------------------------------------------------------------- +DA_CONTEXTS = [DA_CONTEXT_ADMIN] +DA_TOPOLOGIES = [DA_TOPOLOGY_ADMIN] +DA_DEVICES = [DA_DEVICE_DEV1, DA_DEVICE_DEV2, DA_DEVICE_DEV3] +DA_LINKS = [DA_LINK_DEV1_DEV2, DA_LINK_DEV1_DEV3, DA_LINK_DEV2_DEV3] + + +# ===== Domain B ======================================================================================================= + # ----- Context -------------------------------------------------------------------------------------------------------- -CONTEXT_ADMIN_ID = json_context_id(DEFAULT_CONTEXT_UUID) -CONTEXT_ADMIN = json_context(DEFAULT_CONTEXT_UUID) +DB_CONTEXT_ADMIN_ID = json_context_id('B') +DB_CONTEXT_ADMIN = json_context('B') # ----- Topology ------------------------------------------------------------------------------------------------------- -TOPOLOGY_ADMIN_ID = json_topology_id(DEFAULT_TOPOLOGY_UUID, context_id=CONTEXT_ADMIN_ID) -TOPOLOGY_ADMIN = json_topology(DEFAULT_TOPOLOGY_UUID, context_id=CONTEXT_ADMIN_ID) +DB_TOPOLOGY_ADMIN_ID = json_topology_id('B', context_id=DB_CONTEXT_ADMIN_ID) +DB_TOPOLOGY_ADMIN = json_topology('B', context_id=DB_CONTEXT_ADMIN_ID) # ----- Devices -------------------------------------------------------------------------------------------------------- -DEVICE_DEV1_ID, DEVICE_DEV1_ENDPOINTS, DEVICE_DEV1 = compose_device('DEV1', ['1', '2']) -DEVICE_DEV2_ID, DEVICE_DEV2_ENDPOINTS, DEVICE_DEV2 = compose_device('DEV2', ['1', '2']) -DEVICE_DEV3_ID, DEVICE_DEV3_ENDPOINTS, DEVICE_DEV3 = compose_device('DEV3', ['1', '2']) +DB_DEVICE_DEV1_ID, DB_DEVICE_DEV1_ENDPOINTS, DB_DEVICE_DEV1 = compose_device('DEV1@B', ['1', '2']) +DB_DEVICE_DEV2_ID, DB_DEVICE_DEV2_ENDPOINTS, DB_DEVICE_DEV2 = compose_device('DEV2@B', ['1', '2']) +DB_DEVICE_DEV3_ID, DB_DEVICE_DEV3_ENDPOINTS, DB_DEVICE_DEV3 = compose_device('DEV3@B', ['1', '2']) # ----- Links ---------------------------------------------------------------------------------------------------------- -LINK_DEV1_DEV2_ID, LINK_DEV1_DEV2 = compose_link(DEVICE_DEV1_ENDPOINTS[0], DEVICE_DEV2_ENDPOINTS[0]) -LINK_DEV1_DEV3_ID, LINK_DEV1_DEV3 = compose_link(DEVICE_DEV1_ENDPOINTS[1], DEVICE_DEV3_ENDPOINTS[0]) -LINK_DEV2_DEV3_ID, LINK_DEV2_DEV3 = compose_link(DEVICE_DEV2_ENDPOINTS[1], DEVICE_DEV3_ENDPOINTS[1]) +DB_LINK_DEV1_DEV2_ID, DB_LINK_DEV1_DEV2 = compose_link(DB_DEVICE_DEV1_ENDPOINTS[0], DB_DEVICE_DEV2_ENDPOINTS[0]) +DB_LINK_DEV1_DEV3_ID, DB_LINK_DEV1_DEV3 = compose_link(DB_DEVICE_DEV1_ENDPOINTS[1], DB_DEVICE_DEV3_ENDPOINTS[0]) +DB_LINK_DEV2_DEV3_ID, DB_LINK_DEV2_DEV3 = compose_link(DB_DEVICE_DEV2_ENDPOINTS[1], DB_DEVICE_DEV3_ENDPOINTS[1]) # ----- Containers ----------------------------------------------------------------------------------------------------- -CONTEXTS = [CONTEXT_ADMIN] -TOPOLOGIES = [TOPOLOGY_ADMIN] -DEVICES = [DEVICE_DEV1, DEVICE_DEV2, DEVICE_DEV3] -LINKS = [LINK_DEV1_DEV2, LINK_DEV1_DEV3, LINK_DEV2_DEV3] +DB_CONTEXTS = [DB_CONTEXT_ADMIN] +DB_TOPOLOGIES = [DB_TOPOLOGY_ADMIN] +DB_DEVICES = [DB_DEVICE_DEV1, DB_DEVICE_DEV2, DB_DEVICE_DEV3] +DB_LINKS = [DB_LINK_DEV1_DEV2, DB_LINK_DEV1_DEV3, DB_LINK_DEV2_DEV3] diff --git a/src/dlt/connector/tests/__init__.py b/src/dlt/connector/tests/__init__.py index 70a332512..9953c8205 100644 --- a/src/dlt/connector/tests/__init__.py +++ b/src/dlt/connector/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -- GitLab From 2b9be99ab61393e4f11deb6a736ad5082907302f Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Tue, 19 Jul 2022 14:39:39 +0200 Subject: [PATCH 07/36] DLT connector: intermediate backup --- genproto.sh | 7 ++ src/dlt/connector/.gitlab-ci.yml | 2 - .../connector/client/DltEventsCollector.py | 72 +++++++++++++++++ src/dlt/connector/main_test.py | 15 ++++ .../tests/MockService_Dependencies.py | 38 +++++++++ .../connector/tests/PrepareTestScenario.py | 77 ++++++++++++++++--- src/dlt/connector/tests/test_unitary.py | 8 +- 7 files changed, 201 insertions(+), 18 deletions(-) create mode 100755 genproto.sh create mode 100644 src/dlt/connector/client/DltEventsCollector.py create mode 100644 src/dlt/connector/main_test.py create mode 100644 src/dlt/connector/tests/MockService_Dependencies.py diff --git a/genproto.sh b/genproto.sh new file mode 100755 index 000000000..d7ffe50e6 --- /dev/null +++ b/genproto.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eu + +mkdir -p src/common/proto +rm -rf src/common/proto/*.py +touch src/common/proto/__init__.py +python3 -m grpc_tools.protoc -I=./proto --python_out=src/common/proto/ --grpc_python_out=src/common/proto/ proto/*.proto +find src/common/proto -type f -iname *.py -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; diff --git a/src/dlt/connector/.gitlab-ci.yml b/src/dlt/connector/.gitlab-ci.yml index d62e8edad..08c58ae4a 100644 --- a/src/dlt/connector/.gitlab-ci.yml +++ b/src/dlt/connector/.gitlab-ci.yml @@ -16,7 +16,6 @@ build slice: variables: IMAGE_NAME: 'slice' # name of the microservice - IMAGE_NAME_TEST: 'slice-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: @@ -34,7 +33,6 @@ build slice: unit_test slice: variables: IMAGE_NAME: 'slice' # name of the microservice - IMAGE_NAME_TEST: 'slice-test' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: diff --git a/src/dlt/connector/client/DltEventsCollector.py b/src/dlt/connector/client/DltEventsCollector.py new file mode 100644 index 000000000..6fe2474ce --- /dev/null +++ b/src/dlt/connector/client/DltEventsCollector.py @@ -0,0 +1,72 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging, queue, threading +from common.proto.dlt_gateway_pb2 import DltRecordSubscription +from common.tools.grpc.Tools import grpc_message_to_json_string +from dlt.connector.client.DltGatewayClient import DltGatewayClient + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +class DltEventsCollector: + def __init__( + self, dltgateway_client : DltGatewayClient, + log_events_received : bool = False, + ) -> None: + self._events_queue = queue.Queue() + self._log_events_received = log_events_received + subscription = DltRecordSubscription() # bu default subscribe to all + self._dltgateway_stream = dltgateway_client.SubscribeToDlt(subscription) + self._dltgateway_thread = self._create_collector_thread(self._dltgateway_stream) + + def _create_collector_thread(self, stream, as_daemon : bool = False): + return threading.Thread(target=self._collect, args=(stream,), daemon=as_daemon) + + def _collect(self, events_stream) -> None: + try: + for event in events_stream: + if self._log_events_received: + LOGGER.info('[_collect] event: {:s}'.format(grpc_message_to_json_string(event))) + self._events_queue.put_nowait(event) + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.CANCELLED: # pylint: disable=no-member + raise # pragma: no cover + + def start(self): + if self._dltgateway_thread is not None: self._dltgateway_thread.start() + + def get_event(self, block : bool = True, timeout : float = 0.1): + try: + return self._events_queue.get(block=block, timeout=timeout) + except queue.Empty: # pylint: disable=catching-non-exception + return None + + def get_events(self, block : bool = True, timeout : float = 0.1, count : int = None): + events = [] + if count is None: + while True: + event = self.get_event(block=block, timeout=timeout) + if event is None: break + events.append(event) + else: + for _ in range(count): + event = self.get_event(block=block, timeout=timeout) + if event is None: continue + events.append(event) + return sorted(events, key=lambda e: e.event.timestamp.timestamp) + + def stop(self): + if self._dltgateway_stream is not None: self._dltgateway_stream.cancel() + if self._dltgateway_thread is not None: self._dltgateway_thread.join() diff --git a/src/dlt/connector/main_test.py b/src/dlt/connector/main_test.py new file mode 100644 index 000000000..a5ad9c33c --- /dev/null +++ b/src/dlt/connector/main_test.py @@ -0,0 +1,15 @@ +import sys +from .client.DltGatewayClient import DltGatewayClient +from .client.DltEventsCollector import DltEventsCollector + +def main(): + dltgateway_client_1 = DltGatewayClient(host='', port=0) + dltgateway_client_2 = DltGatewayClient(host='', port=0) + dltgateway_client_3 = DltGatewayClient(host='', port=0) + + dltgateway_collector_1 = DltEventsCollector(dltgateway_client_1, log_events_received=True) + dltgateway_collector_2 = DltEventsCollector(dltgateway_client_2, log_events_received=True) + dltgateway_collector_3 = DltEventsCollector(dltgateway_client_3, log_events_received=True) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/dlt/connector/tests/MockService_Dependencies.py b/src/dlt/connector/tests/MockService_Dependencies.py new file mode 100644 index 000000000..65ddc3cb4 --- /dev/null +++ b/src/dlt/connector/tests/MockService_Dependencies.py @@ -0,0 +1,38 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Union +from common.Constants import ServiceNameEnum +from common.Settings import ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name +from common.proto.dlt_gateway_pb2_grpc import add_DltGatewayServiceServicer_to_server +from common.tests.MockServicerImpl_DltGateway import MockServicerImpl_DltGateway +from common.tools.service.GenericGrpcService import GenericGrpcService + +LOCAL_HOST = '127.0.0.1' + +SERVICE_DLT = ServiceNameEnum.DLT + +class MockService_Dependencies(GenericGrpcService): + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.dltgateway_servicer = MockServicerImpl_DltGateway() + add_DltGatewayServiceServicer_to_server(self.dltgateway_servicer, self.server) + + def configure_env_vars(self): + os.environ[get_env_var_name(SERVICE_DLT, ENVVAR_SUFIX_SERVICE_HOST )] = str(self.bind_address) + os.environ[get_env_var_name(SERVICE_DLT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(self.bind_port) diff --git a/src/dlt/connector/tests/PrepareTestScenario.py b/src/dlt/connector/tests/PrepareTestScenario.py index 271deb72a..5c5d1cb5c 100644 --- a/src/dlt/connector/tests/PrepareTestScenario.py +++ b/src/dlt/connector/tests/PrepareTestScenario.py @@ -23,28 +23,87 @@ from common.message_broker.Factory import get_messagebroker_backend, BackendEnum from common.message_broker.MessageBroker import MessageBroker from context.client.ContextClient import ContextClient from context.service.grpc_server.ContextService import ContextService +from dlt.connector.client.DltConnectorClient import DltConnectorClient +from dlt.connector.service.DltConnectorService import DltConnectorService +from .MockService_Dependencies import MockService_Dependencies LOCAL_HOST = '127.0.0.1' -GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT) # avoid privileged ports -os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT) +MOCKSERVICE_PORT = 10000 +#GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT) # avoid privileged ports +#os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +#os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT) + +# ===== BlockChain Emulator (Mock DLT Gateway) ========================================================================= +# A single gateway is used for all the domains + +@pytest.fixture(scope='session') +def dltgateway_service(): + _service = MockService_Dependencies(MOCKSERVICE_PORT) + _service.configure_env_vars() + _service.start() + yield _service + _service.stop() + +# ===== Domain A (Real Context + Real DLT Connector) =================================================================== + +@pytest.fixture(scope='session') +def context_service_a(): # pylint: disable=redefined-outer-name + _database = Database(get_database_backend(backend=DatabaseBackendEnum.INMEMORY)) + _message_broker = MessageBroker(get_messagebroker_backend(backend=MessageBrokerBackendEnum.INMEMORY)) + _service = ContextService(_database, _message_broker) + _service.start() + yield _service + _service.stop() + _message_broker.terminate() + +@pytest.fixture(scope='session') +def context_client_a(context_service_a : ContextService): # pylint: disable=redefined-outer-name + _client = ContextClient(host=context_service_a.bind_address, port=context_service_a.bind_port) + yield _client + _client.close() + +@pytest.fixture(scope='session') +def dltconnector_service_a(): + _service = DltConnectorService() + _service.bind_port += 1 + _service.start() + yield _service + _service.stop() + +@pytest.fixture(scope='session') +def dltconnector_client_a(dltconnector_service_a : DltConnectorService): # pylint: disable=redefined-outer-name + _client = DltConnectorClient(host=dltconnector_service_a.bind_address, port=dltconnector_service_a.bind_port) + yield _client + _client.close() + +# ===== Domain B (Real Context + Real DLT Connector) =================================================================== @pytest.fixture(scope='session') -def context_db_mb() -> Tuple[Database, MessageBroker]: +def context_service_b(): # pylint: disable=redefined-outer-name _database = Database(get_database_backend(backend=DatabaseBackendEnum.INMEMORY)) _message_broker = MessageBroker(get_messagebroker_backend(backend=MessageBrokerBackendEnum.INMEMORY)) - yield _database, _message_broker + _service = ContextService(_database, _message_broker) + _service.start() + yield _service + _service.stop() _message_broker.terminate() @pytest.fixture(scope='session') -def context_service(context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name - _service = ContextService(context_db_mb[0], context_db_mb[1]) +def context_client_b(context_service_b : ContextService): # pylint: disable=redefined-outer-name + _client = ContextClient(host=context_service_b.bind_address, port=context_service_b.bind_port) + yield _client + _client.close() + +@pytest.fixture(scope='session') +def dltconnector_service_b(): + _service = DltConnectorService() + _service.bind_port += 2 _service.start() yield _service _service.stop() @pytest.fixture(scope='session') -def context_client(context_service : ContextService): # pylint: disable=redefined-outer-name - _client = ContextClient() +def dltconnector_client_b(dltconnector_service_b : DltConnectorService): # pylint: disable=redefined-outer-name + _client = DltConnectorClient(host=dltconnector_service_b.bind_address, port=dltconnector_service_b.bind_port) yield _client _client.close() diff --git a/src/dlt/connector/tests/test_unitary.py b/src/dlt/connector/tests/test_unitary.py index 0f1fcb3eb..f5f54798f 100644 --- a/src/dlt/connector/tests/test_unitary.py +++ b/src/dlt/connector/tests/test_unitary.py @@ -16,9 +16,8 @@ import logging from typing import Tuple from common.orm.Database import Database from common.message_broker.MessageBroker import MessageBroker +from common.proto.context_pb2 import Context, ContextId, Device, DeviceId, Link, LinkId, Topology, TopologyId from context.client.ContextClient import ContextClient -from context.client.EventsCollector import EventsCollector -from context.proto.context_pb2 import Context, ContextId, Device, DeviceId, Link, LinkId, Topology, TopologyId from .PrepareTestScenario import context_db_mb, context_service, context_client # pylint: disable=unused-import from .Objects import CONTEXTS, TOPOLOGIES, DEVICES, LINKS @@ -32,11 +31,6 @@ def test_create_events( context_database.clear_all() - events_collector = EventsCollector(context_client) - events_collector.start() - events = events_collector.get_events(block=True, count=4) - events_collector.stop() - for context in CONTEXTS : context_client.SetContext (Context (**context )) for topology in TOPOLOGIES: context_client.SetTopology(Topology(**topology)) for device in DEVICES : context_client.SetDevice (Device (**device )) -- GitLab From bd1af9712485807a110e27986c235cc87903144b Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 6 Jul 2022 17:57:33 +0200 Subject: [PATCH 08/36] Add gRPC code generation --- src/dlt/gateway/build.gradle.kts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/dlt/gateway/build.gradle.kts b/src/dlt/gateway/build.gradle.kts index 8eb0d53fa..78584026c 100644 --- a/src/dlt/gateway/build.gradle.kts +++ b/src/dlt/gateway/build.gradle.kts @@ -39,10 +39,13 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import com.google.protobuf.gradle.generateProtoTasks import com.google.protobuf.gradle.id +import com.google.protobuf.gradle.plugins import com.google.protobuf.gradle.protobuf import com.google.protobuf.gradle.protoc -ext["protobufVersion"] = "3.19.1" +ext["grpcVersion"] = "1.46.0" +ext["grpcKotlinVersion"] = "1.3.0" // CURRENT_GRPC_KOTLIN_VERSION +ext["protobufVersion"] = "3.20.1" ext["ktorVersion"] = "1.6.5" plugins { @@ -62,6 +65,8 @@ repositories { dependencies { testImplementation("org.jetbrains.kotlin:kotlin-test:1.5.31") implementation("org.hyperledger.fabric:fabric-gateway-java:2.2.2") + api("io.grpc:grpc-stub:${rootProject.ext["grpcVersion"]}") + api("io.grpc:grpc-protobuf:${rootProject.ext["grpcVersion"]}") api("com.google.protobuf:protobuf-kotlin:${rootProject.ext["protobufVersion"]}") implementation("io.ktor:ktor-server-core:${rootProject.ext["ktorVersion"]}") implementation("io.ktor:ktor-server-netty:${rootProject.ext["ktorVersion"]}") @@ -97,6 +102,7 @@ sourceSets { main { proto { srcDir("src/main/kotlin/proto") + srcDir("../../../proto") } } } @@ -105,8 +111,20 @@ protobuf { protoc { artifact = "com.google.protobuf:protoc:${rootProject.ext["protobufVersion"]}" } + plugins { + id("grpc") { + artifact = "io.grpc:protoc-gen-grpc-java:${rootProject.ext["grpcVersion"]}" + } + id("grpckt") { + artifact = "io.grpc:protoc-gen-grpc-kotlin:${rootProject.ext["grpcKotlinVersion"]}:jdk8@jar" + } + } generateProtoTasks { all().forEach { + it.plugins { + id("grpc") + id("grpckt") + } it.builtins { id("kotlin") } -- GitLab From 703208adffd63be051264d12bbb788378538291d Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 12 Jul 2022 17:28:20 +0200 Subject: [PATCH 09/36] Rewrite DLT service to gRPC --- proto/dlt_gateway.proto | 4 +- src/dlt/gateway/build.gradle.kts | 41 +- .../config/ca.org1.example.com-cert.pem | 16 +- src/dlt/gateway/config/connection-org1.json | 16 +- src/dlt/gateway/settings.gradle.kts | 2 +- src/dlt/gateway/src/main/kotlin/Main.kt | 165 ++++---- .../src/main/kotlin/fabric/ConnectGateway.kt | 2 +- .../src/main/kotlin/fabric/FabricConnector.kt | 60 ++- .../src/main/kotlin/grpc/FabricServer.kt | 91 +++++ .../src/main/kotlin/grpc/GrpcHandler.kt | 90 +++++ .../gateway/src/main/kotlin/http/Server.kt | 162 -------- .../src/main/kotlin/proto/Config.proto | 28 +- .../src/main/kotlin/proto/context.proto | 374 ++++++++++++++++++ .../gateway/src/main/kotlin/proto/dlt.proto | 97 +++++ .../main/kotlin/proto/kpi_sample_types.proto | 24 ++ 15 files changed, 833 insertions(+), 339 deletions(-) create mode 100644 src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt create mode 100644 src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt delete mode 100644 src/dlt/gateway/src/main/kotlin/http/Server.kt create mode 100644 src/dlt/gateway/src/main/kotlin/proto/context.proto create mode 100644 src/dlt/gateway/src/main/kotlin/proto/dlt.proto create mode 100644 src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto diff --git a/proto/dlt_gateway.proto b/proto/dlt_gateway.proto index b2c1297cc..84fe0fef6 100644 --- a/proto/dlt_gateway.proto +++ b/proto/dlt_gateway.proto @@ -21,8 +21,8 @@ service DltGatewayService { rpc RecordToDlt (DltRecord ) returns ( DltRecordStatus ) {} rpc GetFromDlt (DltRecordId ) returns ( DltRecord ) {} rpc SubscribeToDlt(DltRecordSubscription ) returns (stream DltRecordEvent ) {} - rpc GetDltStatus (context.TeraFlowController) returns ( DltPeerStatus ) {} // NEC is checkig if it is possible - rpc GetDltPeers (context.Empty ) returns ( DltPeerStatusList) {} // NEC is checkig if it is possible + rpc GetDltStatus (context.TeraFlowController) returns ( DltPeerStatus ) {} // NEC is checking if it is possible + rpc GetDltPeers (context.Empty ) returns ( DltPeerStatusList) {} // NEC is checking if it is possible } enum DltRecordTypeEnum { diff --git a/src/dlt/gateway/build.gradle.kts b/src/dlt/gateway/build.gradle.kts index 78584026c..4f905172e 100644 --- a/src/dlt/gateway/build.gradle.kts +++ b/src/dlt/gateway/build.gradle.kts @@ -43,15 +43,15 @@ import com.google.protobuf.gradle.plugins import com.google.protobuf.gradle.protobuf import com.google.protobuf.gradle.protoc -ext["grpcVersion"] = "1.46.0" +ext["grpcVersion"] = "1.47.0" ext["grpcKotlinVersion"] = "1.3.0" // CURRENT_GRPC_KOTLIN_VERSION ext["protobufVersion"] = "3.20.1" ext["ktorVersion"] = "1.6.5" plugins { - kotlin("jvm") version "1.5.31" + kotlin("jvm") version "1.6.21" kotlin("plugin.serialization") version "1.4.21" - id("com.google.protobuf") version "0.8.17" + id("com.google.protobuf") version "0.8.18" application } @@ -59,24 +59,24 @@ group = "eu.neclab" version = "1.0-SNAPSHOT" repositories { + mavenLocal() + google() mavenCentral() } dependencies { - testImplementation("org.jetbrains.kotlin:kotlin-test:1.5.31") - implementation("org.hyperledger.fabric:fabric-gateway-java:2.2.2") - api("io.grpc:grpc-stub:${rootProject.ext["grpcVersion"]}") - api("io.grpc:grpc-protobuf:${rootProject.ext["grpcVersion"]}") - api("com.google.protobuf:protobuf-kotlin:${rootProject.ext["protobufVersion"]}") - implementation("io.ktor:ktor-server-core:${rootProject.ext["ktorVersion"]}") - implementation("io.ktor:ktor-server-netty:${rootProject.ext["ktorVersion"]}") - implementation("io.ktor:ktor-serialization:${rootProject.ext["ktorVersion"]}") - implementation("io.ktor:ktor-client-serialization:${rootProject.ext["ktorVersion"]}") - implementation("io.ktor:ktor-client-core:${rootProject.ext["ktorVersion"]}") - implementation("io.ktor:ktor-client-cio:${rootProject.ext["ktorVersion"]}") - implementation("ch.qos.logback:logback-classic:1.2.5") + implementation(kotlin("stdlib-jdk8")) + testImplementation("org.jetbrains.kotlin:kotlin-test:1.6.21") + implementation("javax.annotation:javax.annotation-api:1.3.2") + implementation("io.grpc:grpc-kotlin-stub:1.3.0") + implementation("io.grpc:grpc-protobuf:1.47.0") + implementation("com.google.protobuf:protobuf-kotlin:3.21.1") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.3") + implementation("org.hyperledger.fabric:fabric-gateway-java:2.2.5") + implementation("ch.qos.logback:logback-classic:1.2.11") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.1") implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:1.3.1") + runtimeOnly("io.grpc:grpc-netty:${rootProject.ext["grpcVersion"]}") } tasks.test { @@ -102,11 +102,18 @@ sourceSets { main { proto { srcDir("src/main/kotlin/proto") - srcDir("../../../proto") } } } +sourceSets { + val main by getting { } + main.java.srcDirs("build/generated/source/proto/main/grpc") + main.java.srcDirs("build/generated/source/proto/main/grpckt") + main.java.srcDirs("build/generated/source/proto/main/java") + main.java.srcDirs("build/generated/source/proto/main/kotlin") +} + protobuf { protoc { artifact = "com.google.protobuf:protoc:${rootProject.ext["protobufVersion"]}" @@ -130,4 +137,4 @@ protobuf { } } } -} \ No newline at end of file +} diff --git a/src/dlt/gateway/config/ca.org1.example.com-cert.pem b/src/dlt/gateway/config/ca.org1.example.com-cert.pem index 5287a0f2b..aca53bf6e 100644 --- a/src/dlt/gateway/config/ca.org1.example.com-cert.pem +++ b/src/dlt/gateway/config/ca.org1.example.com-cert.pem @@ -1,14 +1,14 @@ -----BEGIN CERTIFICATE----- -MIICJjCCAc2gAwIBAgIUWZ4l32loO9+FM0FYw61y3dUF5a0wCgYIKoZIzj0EAwIw +MIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw cDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH EwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh -Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzA1MDk0NDAwWhcNMzcwNzAxMDk0NDAw +Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw WjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV BAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT -Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABNPg -yfDxHr4ZmFp3HB19f27vfc1YTKBnznLqIFwVad2Y+eXfni8DnTRNGgwdkG9uIK2L -4Y9mwlKG/mTNx629G4GjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG -AQH/AgEBMB0GA1UdDgQWBBSZlT6qe+DAGpEBXyMxzidqCkQ4PjAKBggqhkjOPQQD -AgNHADBEAiAIG5jwBGddB9CwocmjAzFv8+e7+0bvNSwjrG229QogTgIgbTNoC33P -mbR5ChlkUAW2t41hTOCSMIwLAlvEwpeCnAk= +Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u +lEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU +4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG +AQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD +AgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6 +H4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz -----END CERTIFICATE----- diff --git a/src/dlt/gateway/config/connection-org1.json b/src/dlt/gateway/config/connection-org1.json index 320a20806..17170a215 100644 --- a/src/dlt/gateway/config/connection-org1.json +++ b/src/dlt/gateway/config/connection-org1.json @@ -24,9 +24,9 @@ }, "peers": { "peer0.org1.example.com": { - "url": "grpcs://s2:7051", + "url": "grpcs://teraflow.nlehd.de:7051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUWZ4l32loO9+FM0FYw61y3dUF5a0wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzA1MDk0NDAwWhcNMzcwNzAxMDk0NDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABNPg\nyfDxHr4ZmFp3HB19f27vfc1YTKBnznLqIFwVad2Y+eXfni8DnTRNGgwdkG9uIK2L\n4Y9mwlKG/mTNx629G4GjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSZlT6qe+DAGpEBXyMxzidqCkQ4PjAKBggqhkjOPQQD\nAgNHADBEAiAIG5jwBGddB9CwocmjAzFv8+e7+0bvNSwjrG229QogTgIgbTNoC33P\nmbR5ChlkUAW2t41hTOCSMIwLAlvEwpeCnAk=\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u\nlEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU\n4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD\nAgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6\nH4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org1.example.com", @@ -34,9 +34,9 @@ } }, "peer0.org2.example.com": { - "url": "grpcs://s2:9051", + "url": "grpcs://teraflow.nlehd.de:9051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICHzCCAcWgAwIBAgIUejv57h6dJkVIM2R1YnlqykkvG7gwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MDUwOTQ0MDBaFw0zNzA3MDEwOTQ0MDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASFZqoisCIgZyMM\n8e0YBA+jxH/+Fc4Y4OkEl5uGRXGl9s0OemCdvhlX9K+esX2DVk1st1PMfTEj/six\n9XPpVqzNo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUPEyzGBlZEjguoJB16wAmoH2bAh8wCgYIKoZIzj0EAwIDSAAwRQIh\nAL6DAWgrqRtbYoQ0oYAr/2vze0JtQcXoqiQKlyvYkUBbAiB/uSHBk3NwjzI8t8iW\nzQzr5eNy5JwOO0SWwPEv4Ev9iQ==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUMyzCixoqXQBzqrCTv301KO5Ub3wwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTIwODQxMDBaFw0zNzA3MDgwODQxMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATpsE9qr9oDDxEZ\nLNDGZduPAslCFl3AY8wC2dURdTJQmn3V/AMeFGiC+a9xEXF5+C/UR/GHE8Cca8/j\nK8GtiM06o0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUE4+hU1/yQ/NCcqrhtAOcSj3O/3EwCgYIKoZIzj0EAwIDRwAwRAIg\nVDsbCz7hPyZ2tULxWiTUnG0aPg7TBh6lD5mRMObX1vUCIA1kcgFnfMPWA1lJJMRQ\nqGw7ydxYTyZ4X4M1b1PPAa3f\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org2.example.com", @@ -46,11 +46,11 @@ }, "certificateAuthorities": { "ca.org1.example.com": { - "url": "https://s2:7054", + "url": "https://teraflow.nlehd.de:7054", "caName": "ca-org1", "tlsCACerts": { "pem": [ - "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUWZ4l32loO9+FM0FYw61y3dUF5a0wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzA1MDk0NDAwWhcNMzcwNzAxMDk0NDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABNPg\nyfDxHr4ZmFp3HB19f27vfc1YTKBnznLqIFwVad2Y+eXfni8DnTRNGgwdkG9uIK2L\n4Y9mwlKG/mTNx629G4GjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSZlT6qe+DAGpEBXyMxzidqCkQ4PjAKBggqhkjOPQQD\nAgNHADBEAiAIG5jwBGddB9CwocmjAzFv8+e7+0bvNSwjrG229QogTgIgbTNoC33P\nmbR5ChlkUAW2t41hTOCSMIwLAlvEwpeCnAk=\n-----END CERTIFICATE-----\n" + "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u\nlEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU\n4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD\nAgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6\nH4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz\n-----END CERTIFICATE-----\n" ] }, "httpOptions": { @@ -60,9 +60,9 @@ }, "orderers": { "orderer0.example.com": { - "url": "grpcs://s2:7050", + "url": "grpcs://teraflow.nlehd.de:7050", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIURV0KgZTOagIAIU7wRcSg/mKl5RUwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcwNTA5NDQwMFoXDTM3MDcwMTA5NDQwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEWrOugtJgVLAKZRw9jaC15RUbVuTm0ZmsqNyiQrKQ\nYawLE6fs+QIU7WQ25fxlYtmGB2S8nofGCDuwaoTevW0GoaNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFLKBzGXaQg2Irr57\npjoFYZ9F1NoNMAoGCCqGSM49BAMCA0cAMEQCIB1YdgOEsATw2GeaFmq6nqWg0JDT\np456JB/reFmnPWdJAiBPo5H9sMh+MpP4R5ue7nuwYK7SEJ1DOJqWMlPuNhVgtA==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIULAsUlO5yAbjj/6xnDbyiWeoj6rkwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxMjA4NDAwMFoXDTM3MDcwODA4NDAwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEjwa9/lOenYmcYL+Kqns6lV+UFt8+3tKZHlcwogZf\ngYnBEbsdL9oTZXSFgIvpCpQee65KKFUl7SqMDXrO3Mg0hqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFPuDBFVg+lhySQO0\nICEnNQ/TzRXeMAoGCCqGSM49BAMCA0cAMEQCIEadLxytDs79F5qTrTuI6SdaFoLB\n6AoQ5VGUNm2mF8tGAiAlWrLs8g3Ni2c7cjFeHZfiFy/2pqJLtor1ieGlFslq1w==\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "orderer0.example.com", diff --git a/src/dlt/gateway/settings.gradle.kts b/src/dlt/gateway/settings.gradle.kts index 0ebdd07b2..67683a744 100644 --- a/src/dlt/gateway/settings.gradle.kts +++ b/src/dlt/gateway/settings.gradle.kts @@ -1,3 +1,3 @@ -rootProject.name = "dlt" +rootProject.name = "gateway" diff --git a/src/dlt/gateway/src/main/kotlin/Main.kt b/src/dlt/gateway/src/main/kotlin/Main.kt index 68a820ee9..468e0a7b9 100644 --- a/src/dlt/gateway/src/main/kotlin/Main.kt +++ b/src/dlt/gateway/src/main/kotlin/Main.kt @@ -12,7 +12,7 @@ // duplication of the object or source code - either totally or in // part - is strictly prohibited. // -// Copyright (c) 2021 NEC Laboratories Europe GmbH +// Copyright (c) 2022 NEC Laboratories Europe GmbH // All Rights Reserved. // // Authors: Konstantin Munichev @@ -35,107 +35,86 @@ // // THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. -import io.ktor.client.* -import io.ktor.client.engine.cio.* -import io.ktor.client.features.* -import io.ktor.client.request.* -import io.ktor.utils.io.jvm.javaio.* -import kotlinx.serialization.ExperimentalSerializationApi -import proto.Config -import proto.Config.DltConfig - -@OptIn(ExperimentalSerializationApi::class) -suspend fun main(args: Array) { - // TODO: default configuration file - val cfg = DltConfig.newBuilder().setWallet("wallet").setConnectionFile("config/connection-org1.json") - .setUser("appUser") - .setChannel("dlt") - .setContract("basic").setCaCertFile("config/ca.org1.example.com-cert.pem").setCaUrl("https://s2:7054") - .setCaAdmin("admin").setCaAdminSecret("adminpw").setMsp("Org1MSP").setAffiliation("org1.department1") - .build() - val cfgBytes = cfg.toByteArray() +import io.grpc.ManagedChannel +import io.grpc.ManagedChannelBuilder +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import proto.ContextOuterClass +import proto.Dlt +import proto.DltServiceGrpcKt +import java.io.Closeable +import java.util.concurrent.TimeUnit - val client = HttpClient(CIO) { - HttpResponseValidator { - validateResponse { response -> - println(response.status) - } - } - } +class DltServiceClient(private val channel: ManagedChannel) : Closeable { + private val stub: DltServiceGrpcKt.DltServiceCoroutineStub = DltServiceGrpcKt.DltServiceCoroutineStub(channel) - try { - client.post("http://localhost:8080/dlt/configure") { - body = cfgBytes - } - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) + suspend fun putData(data: Dlt.DltRecord) { + println("Sending record ${data.recordId}...") + val response = stub.recordToDlt(data) + println("Response: ${response.recordId}") } - try { - val config = client.get("http://localhost:8080/dlt/configure") - println(DltConfig.parseFrom(config)) - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) + suspend fun getData(id: Dlt.DltRecordId) { + println("Requesting record $id...") + val response = stub.getFromDlt(id) + println("Got data: $response") } - val uuid = "41f4d2e2-f4ef-4c81-872a-c32f2d26b2ca" - try { - val record = client.get("http://localhost:8080/dlt/record") { - body = uuid + fun subscribe(filter: Dlt.DltRecordSubscription) { + val subscription = stub.subscribeToDlt(filter) + GlobalScope.launch { + subscription.collect { + println("Got subscription event") + println(it) + } } - println(Config.DltRecord.parseFrom(record)) - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) } - val id = Config.DltRecordId.newBuilder().setUuid(uuid).build() - val record = Config.DltRecord.newBuilder().setId(id).setOperation(Config.DltRecordOperation.ADD) - .setType(Config.DltRecordType.DEVICE).setJson("{}").build() - try { - val result = client.post("http://localhost:8080/dlt/record") { - body = record.toByteArray() - } - println(String(result)) - val requestedRecord = client.get("http://localhost:8080/dlt/record") { - body = uuid - } - println(Config.DltRecord.parseFrom(requestedRecord)) - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) + override fun close() { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS) } +} - try { - val newRecord = Config.DltRecord.newBuilder().setId(id).setOperation(Config.DltRecordOperation.UPDATE) - .setType(Config.DltRecordType.UNKNOWN).setJson("{}").build() - val result = client.post("http://localhost:8080/dlt/record") { - body = newRecord.toByteArray() - } - println(String(result)) - val requestedRecord = client.get("http://localhost:8080/dlt/record") { - body = uuid - } - println(Config.DltRecord.parseFrom(requestedRecord)) - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) - } - try { - val newRecord = Config.DltRecord.newBuilder().setId(id).setOperation(Config.DltRecordOperation.DISABLE).build() - val result = client.post("http://localhost:8080/dlt/record") { - body = newRecord.toByteArray() - } - println(String(result)) - val requestedRecord = client.get("http://localhost:8080/dlt/record") { - body = uuid - } - println(Config.DltRecord.parseFrom(requestedRecord)) - } catch (e: ClientRequestException) { - println(e.response.status) - println(String(e.response.content.toInputStream().readAllBytes())) - } -} \ No newline at end of file +fun main() = runBlocking { + val port = 50051 + val channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build() + + val client = DltServiceClient(channel) + + val id = Dlt.DltRecordId.newBuilder() + .setDomainUuid( + ContextOuterClass.Uuid.newBuilder() + .setUuid("41f4d2e2-f4ef-4c81-872a-c32f2d26b2ca") + ) + .setRecordUuid( + ContextOuterClass.Uuid.newBuilder() + .setUuid("b4e698d6-2dca-41ea-8483-263d40372071") + ) + .setType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) + .build() + + val subscription = Dlt.DltRecordSubscription.newBuilder() + .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_CONTEXT) + .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_LINK) + .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) + .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) + .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) + .build() + + client.subscribe(subscription) + + Thread.sleep(5000) + + val data = Dlt.DltRecord.newBuilder() + .setRecordId(id) + .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) + .setDataJson("{}") + .build() + + client.putData(data) + client.getData(id) + + Thread.sleep(5000) +} diff --git a/src/dlt/gateway/src/main/kotlin/fabric/ConnectGateway.kt b/src/dlt/gateway/src/main/kotlin/fabric/ConnectGateway.kt index 245bd4828..00ec40d57 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/ConnectGateway.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/ConnectGateway.kt @@ -12,7 +12,7 @@ // duplication of the object or source code - either totally or in // part - is strictly prohibited. // -// Copyright (c) 2021 NEC Laboratories Europe GmbH +// Copyright (c) 2022 NEC Laboratories Europe GmbH // All Rights Reserved. // // Authors: Konstantin Munichev diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index d7c163954..b6200f3f0 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -12,7 +12,7 @@ // duplication of the object or source code - either totally or in // part - is strictly prohibited. // -// Copyright (c) 2021 NEC Laboratories Europe GmbH +// Copyright (c) 2022 NEC Laboratories Europe GmbH // All Rights Reserved. // // Authors: Konstantin Munichev @@ -37,6 +37,9 @@ package fabric +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.runBlocking +import proto.Dlt import org.hyperledger.fabric.gateway.Contract import org.hyperledger.fabric.gateway.ContractEvent import org.hyperledger.fabric.gateway.Wallet @@ -53,6 +56,8 @@ class FabricConnector(val config: Config.DltConfig) { private val wallet: Wallet private val contract: Contract + private val channels: MutableList> = mutableListOf() + init { // Create a CA client for interacting with the CA. val props = Properties() @@ -65,7 +70,24 @@ class FabricConnector(val config: Config.DltConfig) { // Create a wallet for managing identities wallet = Wallets.newFileSystemWallet(Paths.get(config.wallet)) contract = connect() - subscribeForEvents() + + fabricSubscribe() + } + + private fun fabricSubscribe() { + val consumer = Consumer { event: ContractEvent? -> + run { + println("new event detected") + val parsedEvent = Dlt.DltRecordEvent.parseFrom(event?.payload?.get()) + println(parsedEvent.recordId.recordUuid) + runBlocking { + channels.forEach { + it.trySend(parsedEvent) + } + } + } + } + contract.addContractListener(consumer) } fun connect(): Contract { @@ -74,30 +96,31 @@ class FabricConnector(val config: Config.DltConfig) { return getContract(config, wallet) } - fun putData(record: Config.DltRecord): String { - println(record.type.toString()) + fun putData(record: Dlt.DltRecord): String { + println(record.toString()) return String( contract.submitTransaction( "AddRecord", - record.id.uuid, - record.type.number.toString(), - record.json + record.recordId.domainUuid.uuid, + record.recordId.recordUuid.uuid, + record.recordId.type.number.toString(), + record.dataJson ) ) } - fun getData(uuid: String): Config.DltRecord { + fun getData(uuid: String): Dlt.DltRecord { val result = contract.evaluateTransaction("GetRecord", uuid) - return Config.DltRecord.parseFrom(result) + return Dlt.DltRecord.parseFrom(result) } - fun updateData(record: Config.DltRecord): String { + fun updateData(record: Dlt.DltRecord): String { return String( contract.submitTransaction( "UpdateRecord", - record.id.uuid, - record.type.number.toString(), - record.json + record.recordId.recordUuid.uuid, + record.recordId.type.number.toString(), + record.dataJson ) ) } @@ -106,12 +129,9 @@ class FabricConnector(val config: Config.DltConfig) { return String(contract.submitTransaction("DeactivateRecord", uuid)) } - private fun subscribeForEvents() { - val consumer = Consumer { - event: ContractEvent? -> run { - println(event?.payload?.get()?.let { String(it) }) - } - } - contract.addContractListener(consumer) + fun subscribeForEvents(): Channel { + val produceCh = Channel() + channels.add(produceCh) + return produceCh } } \ No newline at end of file diff --git a/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt new file mode 100644 index 000000000..6be4219a7 --- /dev/null +++ b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt @@ -0,0 +1,91 @@ +// NEC Laboratories Europe GmbH +// +// PROPRIETARY INFORMATION +// +// The software and its source code contain valuable trade secrets and +// shall be maintained in confidence and treated as confidential +// information. The software may only be used for evaluation and/or +// testing purposes, unless otherwise explicitly stated in a written +// agreement with NEC Laboratories Europe GmbH. +// +// Any unauthorized publication, transfer to third parties or +// duplication of the object or source code - either totally or in +// part - is strictly prohibited. +// +// Copyright (c) 2022 NEC Laboratories Europe GmbH +// All Rights Reserved. +// +// Authors: Konstantin Munichev +// +// +// NEC Laboratories Europe GmbH DISCLAIMS ALL WARRANTIES, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE AND THE +// WARRANTY AGAINST LATENT DEFECTS, WITH RESPECT TO THE PROGRAM AND +// THE ACCOMPANYING DOCUMENTATION. +// +// NO LIABILITIES FOR CONSEQUENTIAL DAMAGES: IN NO EVENT SHALL NEC +// Laboratories Europe GmbH or ANY OF ITS SUBSIDIARIES BE LIABLE FOR +// ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR +// LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF +// INFORMATION, OR OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, +// INCIDENTAL, ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF +// OR INABILITY TO USE THIS PROGRAM, EVEN IF NEC Laboratories Europe +// GmbH HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +// +// THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. + +package grpc + +import fabric.FabricConnector +import io.grpc.Server +import io.grpc.ServerBuilder +import proto.Config + +class FabricServer(val port: Int) { + private val server: Server + + init { + val cfg = Config.DltConfig.newBuilder().setWallet("wallet").setConnectionFile("config/connection-org1.json") + .setUser("appUser") + .setChannel("dlt") + .setContract("basic").setCaCertFile("config/ca.org1.example.com-cert.pem").setCaUrl("https://teraflow.nlehd.de:7054") + .setCaAdmin("admin").setCaAdminSecret("adminpw").setMsp("Org1MSP").setAffiliation("org1.department1") + .build() + val connector = FabricConnector(cfg) + + val dltService = DLTService(connector) + server = ServerBuilder + .forPort(port) + .addService(dltService) + .build() + + } + + fun start() { + server.start() + println("Server started, listening on $port") + Runtime.getRuntime().addShutdownHook( + Thread { + println("Shutting down...") + this@FabricServer.stop() + println("Server shut down") + } + ) + } + + private fun stop() { + server.shutdown() + } + + fun blockUntilShutdown() { + server.awaitTermination() + } +} + +fun main() { + val port = 50051 + val server = FabricServer(port) + server.start() + server.blockUntilShutdown() +} diff --git a/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt new file mode 100644 index 000000000..ae1b35faa --- /dev/null +++ b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt @@ -0,0 +1,90 @@ +// NEC Laboratories Europe GmbH +// +// PROPRIETARY INFORMATION +// +// The software and its source code contain valuable trade secrets and +// shall be maintained in confidence and treated as confidential +// information. The software may only be used for evaluation and/or +// testing purposes, unless otherwise explicitly stated in a written +// agreement with NEC Laboratories Europe GmbH. +// +// Any unauthorized publication, transfer to third parties or +// duplication of the object or source code - either totally or in +// part - is strictly prohibited. +// +// Copyright (c) 2022 NEC Laboratories Europe GmbH +// All Rights Reserved. +// +// Authors: Konstantin Munichev +// +// +// NEC Laboratories Europe GmbH DISCLAIMS ALL WARRANTIES, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE AND THE +// WARRANTY AGAINST LATENT DEFECTS, WITH RESPECT TO THE PROGRAM AND +// THE ACCOMPANYING DOCUMENTATION. +// +// NO LIABILITIES FOR CONSEQUENTIAL DAMAGES: IN NO EVENT SHALL NEC +// Laboratories Europe GmbH or ANY OF ITS SUBSIDIARIES BE LIABLE FOR +// ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR +// LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF +// INFORMATION, OR OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, +// INCIDENTAL, ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF +// OR INABILITY TO USE THIS PROGRAM, EVEN IF NEC Laboratories Europe +// GmbH HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +// +// THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. + +package grpc + +import proto.Dlt +import fabric.FabricConnector +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import proto.ContextOuterClass +import proto.DltServiceGrpcKt + +class DLTService(private val connector: FabricConnector): DltServiceGrpcKt.DltServiceCoroutineImplBase() { + override suspend fun recordToDlt(request: Dlt.DltRecord): Dlt.DltRecordStatus { + println("Incoming request ${request.recordId.recordUuid}") + val error = connector.putData(request) + val dltStatusEnum: Dlt.DltRecordStatusEnum = if (error == "") { + Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_SUCCEEDED + } else { + Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_FAILED + } + println("Put data status: $error") + return Dlt.DltRecordStatus.newBuilder() + .setRecordId(request.recordId) + .setStatus(dltStatusEnum) + .setErrorMessage(error) + .build() + } + + override suspend fun getFromDlt(request: Dlt.DltRecordId): Dlt.DltRecord { + return connector.getData(request.recordUuid.uuid) + } + + override fun subscribeToDlt(request: Dlt.DltRecordSubscription): Flow = flow { + println("Subscription request: $request") + val c = connector.subscribeForEvents() + println("Got channel") + val event = c.receive() + println("Received event from channel") + + // TODO: mistype, proto scheme fix required + if (request.operationValueList.contains(event.event.eventType.number)) { + if (request.typeList.contains(event.recordId.type)) { + emit(event) + } + } + } + + override suspend fun getDltStatus(request: ContextOuterClass.TeraFlowController): Dlt.DltPeerStatus { + return super.getDltStatus(request) + } + + override suspend fun getDltPeers(request: ContextOuterClass.Empty): Dlt.DltPeerStatusList { + return super.getDltPeers(request) + } +} \ No newline at end of file diff --git a/src/dlt/gateway/src/main/kotlin/http/Server.kt b/src/dlt/gateway/src/main/kotlin/http/Server.kt deleted file mode 100644 index 4e3400af3..000000000 --- a/src/dlt/gateway/src/main/kotlin/http/Server.kt +++ /dev/null @@ -1,162 +0,0 @@ -// NEC Laboratories Europe GmbH -// -// PROPRIETARY INFORMATION -// -// The software and its source code contain valuable trade secrets and -// shall be maintained in confidence and treated as confidential -// information. The software may only be used for evaluation and/or -// testing purposes, unless otherwise explicitly stated in a written -// agreement with NEC Laboratories Europe GmbH. -// -// Any unauthorized publication, transfer to third parties or -// duplication of the object or source code - either totally or in -// part - is strictly prohibited. -// -// Copyright (c) 2021 NEC Laboratories Europe GmbH -// All Rights Reserved. -// -// Authors: Konstantin Munichev -// -// -// NEC Laboratories Europe GmbH DISCLAIMS ALL WARRANTIES, EITHER -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES -// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE AND THE -// WARRANTY AGAINST LATENT DEFECTS, WITH RESPECT TO THE PROGRAM AND -// THE ACCOMPANYING DOCUMENTATION. -// -// NO LIABILITIES FOR CONSEQUENTIAL DAMAGES: IN NO EVENT SHALL NEC -// Laboratories Europe GmbH or ANY OF ITS SUBSIDIARIES BE LIABLE FOR -// ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR -// LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF -// INFORMATION, OR OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, -// INCIDENTAL, ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF -// OR INABILITY TO USE THIS PROGRAM, EVEN IF NEC Laboratories Europe -// GmbH HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. -// -// THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. - -package http - -import fabric.FabricConnector -import io.ktor.application.* -import io.ktor.features.* -import io.ktor.http.* -import io.ktor.request.* -import io.ktor.response.* -import io.ktor.routing.* -import io.ktor.server.engine.* -import io.ktor.server.netty.* -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.withContext -import proto.Config -import proto.Config.DltConfig -import proto.Config.DltRecord - -class Server { - var connector: FabricConnector? = null - val port = 8080 - val mutex = Mutex() -} - -fun checkException(e: Exception): String { - if (e.message == null) return "" - return e.message!! -} - -fun main() { - val server = Server() - embeddedServer(Netty, port = server.port) { - install(ContentNegotiation) - routing { - post("/dlt/configure") { - withContext(Dispatchers.IO) { - try { - val data = call.receiveStream() - val config = DltConfig.parseFrom(data) - println(config) - server.mutex.lock() - server.connector = FabricConnector(config) - server.mutex.unlock() - call.response.status(HttpStatusCode.Created) - } - // TODO: catch exceptions one by one - catch (e: Exception) { - call.respond(HttpStatusCode.BadRequest, checkException(e)) - e.printStackTrace() - } - } - } - get("/dlt/configure") { - withContext(Dispatchers.IO) { - server.mutex.lock() - if (server.connector == null) { - server.mutex.unlock() - call.respond(HttpStatusCode.NotFound, "Not initialized") - } else { - val configBytes = server.connector!!.config.toByteArray() - server.mutex.unlock() - call.respond(HttpStatusCode.OK, configBytes) - } - } - } - post("/dlt/record") { - withContext(Dispatchers.IO) { - server.mutex.lock() - try { - if (server.connector == null) { - call.respond(HttpStatusCode.NotFound, "Not initialized") - } else { - val record = DltRecord.parseFrom(call.receiveStream()) - when (record.operation) { - Config.DltRecordOperation.ADD -> { - val result = server.connector!!.putData(record) - call.respond(HttpStatusCode.Created, result) - } - Config.DltRecordOperation.UPDATE -> { - val result = server.connector!!.updateData(record) - call.respond(HttpStatusCode.OK, result) - } - // TODO: Disable should require only uuid - Config.DltRecordOperation.DISABLE -> { - val result = server.connector!!.deleteData(record.id.uuid) - call.respond(HttpStatusCode.OK, result) - } - else -> { - call.respond(HttpStatusCode.BadRequest, "Invalid operation") - } - } - } - } - // TODO: catch exceptions one by one - catch (e: Exception) { - call.respond(HttpStatusCode.BadRequest, checkException(e)) - e.printStackTrace() - } - server.mutex.unlock() - } - } - get("/dlt/record") { - withContext(Dispatchers.IO) { - server.mutex.lock() - try { - if (server.connector == null) { - call.respond(HttpStatusCode.NotFound) - } else { - val uuid = call.receiveText() - println("Uuid request: $uuid") - val result = server.connector!!.getData(uuid) - call.respond(HttpStatusCode.OK, result.toByteArray()) - } - } - // TODO: catch exceptions one by one - catch (e: Exception) { - call.respond(HttpStatusCode.NotFound, checkException(e)) - e.printStackTrace() - } - server.mutex.unlock() - } - } - } - }.start(wait = true) -} diff --git a/src/dlt/gateway/src/main/kotlin/proto/Config.proto b/src/dlt/gateway/src/main/kotlin/proto/Config.proto index f492e63ce..b6d4c5614 100644 --- a/src/dlt/gateway/src/main/kotlin/proto/Config.proto +++ b/src/dlt/gateway/src/main/kotlin/proto/Config.proto @@ -12,7 +12,7 @@ // duplication of the object or source code - either totally or in // part - is strictly prohibited. // -// Copyright (c) 2021 NEC Laboratories Europe GmbH +// Copyright (c) 2022 NEC Laboratories Europe GmbH // All Rights Reserved. // // Authors: Konstantin Munichev @@ -52,29 +52,3 @@ message DltConfig { string msp = 10; string affiliation = 11; } - -message DltRecordId { - string uuid = 1; -} - -enum DltRecordOperation { - OP_UNSET = 0; - ADD = 1; - UPDATE = 2; - DISABLE = 3; -} - -enum DltRecordType { - RECORD_UNSET = 0; - UNKNOWN = 1; - SERVICE = 2; - DEVICE = 3; - SLICE = 4; -} - -message DltRecord { - DltRecordId id = 1; - DltRecordOperation operation = 2; - DltRecordType type = 3; - string json = 4; -} \ No newline at end of file diff --git a/src/dlt/gateway/src/main/kotlin/proto/context.proto b/src/dlt/gateway/src/main/kotlin/proto/context.proto new file mode 100644 index 000000000..a7185ba10 --- /dev/null +++ b/src/dlt/gateway/src/main/kotlin/proto/context.proto @@ -0,0 +1,374 @@ +// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package proto; + +import "kpi_sample_types.proto"; + +service ContextService { + rpc ListContextIds (Empty ) returns ( ContextIdList ) {} + rpc ListContexts (Empty ) returns ( ContextList ) {} + rpc GetContext (ContextId ) returns ( Context ) {} + rpc SetContext (Context ) returns ( ContextId ) {} + rpc RemoveContext (ContextId ) returns ( Empty ) {} + rpc GetContextEvents (Empty ) returns (stream ContextEvent ) {} + + rpc ListTopologyIds (ContextId ) returns ( TopologyIdList ) {} + rpc ListTopologies (ContextId ) returns ( TopologyList ) {} + rpc GetTopology (TopologyId ) returns ( Topology ) {} + rpc SetTopology (Topology ) returns ( TopologyId ) {} + rpc RemoveTopology (TopologyId ) returns ( Empty ) {} + rpc GetTopologyEvents (Empty ) returns (stream TopologyEvent ) {} + + rpc ListDeviceIds (Empty ) returns ( DeviceIdList ) {} + rpc ListDevices (Empty ) returns ( DeviceList ) {} + rpc GetDevice (DeviceId ) returns ( Device ) {} + rpc SetDevice (Device ) returns ( DeviceId ) {} + rpc RemoveDevice (DeviceId ) returns ( Empty ) {} + rpc GetDeviceEvents (Empty ) returns (stream DeviceEvent ) {} + + rpc ListLinkIds (Empty ) returns ( LinkIdList ) {} + rpc ListLinks (Empty ) returns ( LinkList ) {} + rpc GetLink (LinkId ) returns ( Link ) {} + rpc SetLink (Link ) returns ( LinkId ) {} + rpc RemoveLink (LinkId ) returns ( Empty ) {} + rpc GetLinkEvents (Empty ) returns (stream LinkEvent ) {} + + rpc ListServiceIds (ContextId ) returns ( ServiceIdList ) {} + rpc ListServices (ContextId ) returns ( ServiceList ) {} + rpc GetService (ServiceId ) returns ( Service ) {} + rpc SetService (Service ) returns ( ServiceId ) {} + rpc RemoveService (ServiceId ) returns ( Empty ) {} + rpc GetServiceEvents (Empty ) returns (stream ServiceEvent ) {} + + rpc ListSliceIds (ContextId ) returns ( SliceIdList ) {} + rpc ListSlices (ContextId ) returns ( SliceList ) {} + rpc GetSlice (SliceId ) returns ( Slice ) {} + rpc SetSlice (Slice ) returns ( SliceId ) {} + rpc RemoveSlice (SliceId ) returns ( Empty ) {} + rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} + + rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} + rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} + rpc GetConnection (ConnectionId) returns ( Connection ) {} + rpc SetConnection (Connection ) returns ( ConnectionId ) {} + rpc RemoveConnection (ConnectionId) returns ( Empty ) {} + rpc GetConnectionEvents(Empty ) returns (stream ConnectionEvent ) {} +} + +// ----- Generic ------------------------------------------------------------------------------------------------------- +message Empty {} + +message Uuid { + string uuid = 1; +} + +enum EventTypeEnum { + EVENTTYPE_UNDEFINED = 0; + EVENTTYPE_CREATE = 1; + EVENTTYPE_UPDATE = 2; + EVENTTYPE_REMOVE = 3; +} + +message Event { + double timestamp = 1; + EventTypeEnum event_type = 2; +} + +// ----- Context ------------------------------------------------------------------------------------------------------- +message ContextId { + Uuid context_uuid = 1; +} + +message Context { + ContextId context_id = 1; + repeated TopologyId topology_ids = 2; + repeated ServiceId service_ids = 3; + TeraFlowController controller = 4; +} + +message ContextIdList { + repeated ContextId context_ids = 1; +} + +message ContextList { + repeated Context contexts = 1; +} + +message ContextEvent { + Event event = 1; + ContextId context_id = 2; +} + + +// ----- Topology ------------------------------------------------------------------------------------------------------ +message TopologyId { + ContextId context_id = 1; + Uuid topology_uuid = 2; +} + +message Topology { + TopologyId topology_id = 1; + repeated DeviceId device_ids = 2; + repeated LinkId link_ids = 3; +} + +message TopologyIdList { + repeated TopologyId topology_ids = 1; +} + +message TopologyList { + repeated Topology topologies = 1; +} + +message TopologyEvent { + Event event = 1; + TopologyId topology_id = 2; +} + + +// ----- Device -------------------------------------------------------------------------------------------------------- +message DeviceId { + Uuid device_uuid = 1; +} + +message Device { + DeviceId device_id = 1; + string device_type = 2; + DeviceConfig device_config = 3; + DeviceOperationalStatusEnum device_operational_status = 4; + repeated DeviceDriverEnum device_drivers = 5; + repeated EndPoint device_endpoints = 6; +} + +message DeviceConfig { + repeated ConfigRule config_rules = 1; +} + +enum DeviceDriverEnum { + DEVICEDRIVER_UNDEFINED = 0; // also used for emulated + DEVICEDRIVER_OPENCONFIG = 1; + DEVICEDRIVER_TRANSPORT_API = 2; + DEVICEDRIVER_P4 = 3; + DEVICEDRIVER_IETF_NETWORK_TOPOLOGY = 4; + DEVICEDRIVER_ONF_TR_352 = 5; +} + +enum DeviceOperationalStatusEnum { + DEVICEOPERATIONALSTATUS_UNDEFINED = 0; + DEVICEOPERATIONALSTATUS_DISABLED = 1; + DEVICEOPERATIONALSTATUS_ENABLED = 2; +} + +message DeviceIdList { + repeated DeviceId device_ids = 1; +} + +message DeviceList { + repeated Device devices = 1; +} + +message DeviceEvent { + Event event = 1; + DeviceId device_id = 2; +} + + +// ----- Link ---------------------------------------------------------------------------------------------------------- +message LinkId { + Uuid link_uuid = 1; +} + +message Link { + LinkId link_id = 1; + repeated EndPointId link_endpoint_ids = 2; +} + +message LinkIdList { + repeated LinkId link_ids = 1; +} + +message LinkList { + repeated Link links = 1; +} + +message LinkEvent { + Event event = 1; + LinkId link_id = 2; +} + + +// ----- Service ------------------------------------------------------------------------------------------------------- +message ServiceId { + ContextId context_id = 1; + Uuid service_uuid = 2; +} + +message Service { + ServiceId service_id = 1; + ServiceTypeEnum service_type = 2; + repeated EndPointId service_endpoint_ids = 3; + repeated Constraint service_constraints = 4; + ServiceStatus service_status = 5; + ServiceConfig service_config = 6; +} + +enum ServiceTypeEnum { + SERVICETYPE_UNKNOWN = 0; + SERVICETYPE_L3NM = 1; + SERVICETYPE_L2NM = 2; + SERVICETYPE_TAPI_CONNECTIVITY_SERVICE = 3; +} + +enum ServiceStatusEnum { + SERVICESTATUS_UNDEFINED = 0; + SERVICESTATUS_PLANNED = 1; + SERVICESTATUS_ACTIVE = 2; + SERVICESTATUS_PENDING_REMOVAL = 3; +} + +message ServiceStatus { + ServiceStatusEnum service_status = 1; +} + +message ServiceConfig { + repeated ConfigRule config_rules = 1; +} + +message ServiceIdList { + repeated ServiceId service_ids = 1; +} + +message ServiceList { + repeated Service services = 1; +} + +message ServiceEvent { + Event event = 1; + ServiceId service_id = 2; +} + +// ----- Slice --------------------------------------------------------------------------------------------------------- +message SliceId { + ContextId context_id = 1; + Uuid slice_uuid = 2; +} + +message Slice { + SliceId slice_id = 1; + repeated EndPointId slice_endpoint_ids = 2; + repeated Constraint slice_constraints = 3; + repeated ServiceId slice_service_ids = 4; + repeated SliceId slice_subslice_ids = 5; + SliceStatus slice_status = 6; +} + +enum SliceStatusEnum { + SLICESTATUS_UNDEFINED = 0; + SLICESTATUS_PLANNED = 1; + SLICESTATUS_INIT = 2; + SLICESTATUS_ACTIVE = 3; + SLICESTATUS_DEINIT = 4; +} + +message SliceStatus { + SliceStatusEnum slice_status = 1; +} + +message SliceIdList { + repeated SliceId slice_ids = 1; +} + +message SliceList { + repeated Slice slices = 1; +} + +message SliceEvent { + Event event = 1; + SliceId slice_id = 2; +} + +// ----- Connection ---------------------------------------------------------------------------------------------------- +message ConnectionId { + Uuid connection_uuid = 1; +} + +message Connection { + ConnectionId connection_id = 1; + ServiceId service_id = 2; + repeated EndPointId path_hops_endpoint_ids = 3; + repeated ServiceId sub_service_ids = 4; +} + +message ConnectionIdList { + repeated ConnectionId connection_ids = 1; +} + +message ConnectionList { + repeated Connection connections = 1; +} + +message ConnectionEvent { + Event event = 1; + ConnectionId connection_id = 2; +} + + +// ----- Endpoint ------------------------------------------------------------------------------------------------------ +message EndPointId { + TopologyId topology_id = 1; + DeviceId device_id = 2; + Uuid endpoint_uuid = 3; +} + +message EndPoint { + EndPointId endpoint_id = 1; + string endpoint_type = 2; + repeated KpiSampleType kpi_sample_types = 3; +} + + +// ----- Configuration ------------------------------------------------------------------------------------------------- +enum ConfigActionEnum { + CONFIGACTION_UNDEFINED = 0; + CONFIGACTION_SET = 1; + CONFIGACTION_DELETE = 2; +} + +message ConfigRule { + ConfigActionEnum action = 1; + string resource_key = 2; + string resource_value = 3; +} + + +// ----- Constraint ---------------------------------------------------------------------------------------------------- +message Constraint { + string constraint_type = 1; + string constraint_value = 2; +} + + +// ----- Miscellaneous ------------------------------------------------------------------------------------------------- +message TeraFlowController { + ContextId context_id = 1; + string ip_address = 2; + uint32 port = 3; +} + +message AuthenticationResult { + ContextId context_id = 1; + bool authenticated = 2; +} diff --git a/src/dlt/gateway/src/main/kotlin/proto/dlt.proto b/src/dlt/gateway/src/main/kotlin/proto/dlt.proto new file mode 100644 index 000000000..7b145d2bc --- /dev/null +++ b/src/dlt/gateway/src/main/kotlin/proto/dlt.proto @@ -0,0 +1,97 @@ +// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package proto; + +import "context.proto"; + +service DltService { + rpc RecordToDlt (DltRecord ) returns ( DltRecordStatus ) {} + rpc GetFromDlt (DltRecordId ) returns ( DltRecord ) {} + rpc SubscribeToDlt(DltRecordSubscription ) returns (stream DltRecordEvent ) {} + rpc GetDltStatus (TeraFlowController ) returns ( DltPeerStatus ) {} // NEC is checking if it is possible + rpc GetDltPeers (Empty ) returns ( DltPeerStatusList) {} // NEC is checking if it is possible +} + +enum DltRecordTypeEnum { + DLTRECORDTYPE_UNDEFINED = 0; + DLTRECORDTYPE_CONTEXT = 1; + DLTRECORDTYPE_TOPOLOGY = 2; + DLTRECORDTYPE_DEVICE = 3; + DLTRECORDTYPE_LINK = 4; + DLTRECORDTYPE_SERVICE = 5; + DLTRECORDTYPE_SLICE = 6; +} + +enum DltRecordOperationEnum { + DLTRECORDOPERATION_UNDEFINED = 0; + DLTRECORDOPERATION_ADD = 1; + DLTRECORDOPERATION_UPDATE = 2; + DLTRECORDOPERATION_DELETE = 3; +} + +enum DltRecordStatusEnum { + DLTRECORDSTATUS_UNDEFINED = 0; + DLTRECORDSTATUS_SUCCEEDED = 1; + DLTRECORDSTATUS_FAILED = 2; +} + +enum DltStatusEnum { + DLTSTATUS_UNDEFINED = 0; + DLTSTATUS_NOTAVAILABLE = 1; + DLTSTATUS_INITIALIZED = 2; + DLTSTATUS_AVAILABLE = 3; + DLTSTATUS_DEINIT = 4; +} + +message DltRecordId { + Uuid domain_uuid = 1; // unique identifier of domain owning the record + DltRecordTypeEnum type = 2; // type of record + Uuid record_uuid = 3; // unique identifier of the record within the domain context_uuid/topology_uuid +} + +message DltRecord { + DltRecordId record_id = 1; // record identifier + DltRecordOperationEnum operation = 2; // operation to be performed over the record + string data_json = 3; // record content: JSON-encoded record content +} + +message DltRecordSubscription { + // retrieved events have to match ALL conditions. + // i.e., type in types requested, AND operation in operations requested + // TODO: consider adding a more sophisticated filtering + repeated DltRecordTypeEnum type = 1; // selected event types, empty=all + repeated DltRecordOperationEnum operation = 2; // selected event operations, empty=all +} + +message DltRecordEvent { + Event event = 1; // common event data (timestamp & event_type) + DltRecordId record_id = 2; // record identifier associated with this event +} + +message DltRecordStatus { + DltRecordId record_id = 1; // identifier of the associated record + DltRecordStatusEnum status = 2; // status of the record + string error_message = 3; // error message in case of failure, empty otherwise +} + +message DltPeerStatus { + TeraFlowController controller = 1; // Identifier of the TeraFlow controller instance + DltStatusEnum status = 2; // Status of the TeraFlow controller instance +} + +message DltPeerStatusList { + repeated DltPeerStatus peers = 1; // List of peers and their status +} diff --git a/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto b/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto new file mode 100644 index 000000000..82ef51e80 --- /dev/null +++ b/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto @@ -0,0 +1,24 @@ +// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package proto; + +enum KpiSampleType { + KPISAMPLETYPE_UNKNOWN = 0; + KPISAMPLETYPE_PACKETS_TRANSMITTED = 101; + KPISAMPLETYPE_PACKETS_RECEIVED = 102; + KPISAMPLETYPE_BYTES_TRANSMITTED = 201; + KPISAMPLETYPE_BYTES_RECEIVED = 202; +} -- GitLab From 88bdde07009d0127b222c61cc8e33ab856bbd132 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 13 Jul 2022 11:54:12 +0200 Subject: [PATCH 10/36] Add runServer task to start the server --- src/dlt/gateway/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dlt/gateway/build.gradle.kts b/src/dlt/gateway/build.gradle.kts index 4f905172e..92d0b6bc7 100644 --- a/src/dlt/gateway/build.gradle.kts +++ b/src/dlt/gateway/build.gradle.kts @@ -98,6 +98,12 @@ application { mainClass.set("MainKt") } +task("runServer", JavaExec::class) { + main = "grpc.FabricServerKt" + classpath = sourceSets["main"].runtimeClasspath +} + + sourceSets { main { proto { -- GitLab From a3bd8d30ec09238594bcd64d3f88156c9499808c Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 13 Jul 2022 12:05:44 +0200 Subject: [PATCH 11/36] Generate random Uuids --- src/dlt/gateway/src/main/kotlin/Main.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dlt/gateway/src/main/kotlin/Main.kt b/src/dlt/gateway/src/main/kotlin/Main.kt index 468e0a7b9..dc460b8fa 100644 --- a/src/dlt/gateway/src/main/kotlin/Main.kt +++ b/src/dlt/gateway/src/main/kotlin/Main.kt @@ -44,6 +44,7 @@ import proto.ContextOuterClass import proto.Dlt import proto.DltServiceGrpcKt import java.io.Closeable +import java.util.* import java.util.concurrent.TimeUnit class DltServiceClient(private val channel: ManagedChannel) : Closeable { @@ -83,14 +84,17 @@ fun main() = runBlocking { val client = DltServiceClient(channel) + val domainUuid = UUID.randomUUID().toString() + val recordUuid = UUID.randomUUID().toString() + val id = Dlt.DltRecordId.newBuilder() .setDomainUuid( ContextOuterClass.Uuid.newBuilder() - .setUuid("41f4d2e2-f4ef-4c81-872a-c32f2d26b2ca") + .setUuid(domainUuid) ) .setRecordUuid( ContextOuterClass.Uuid.newBuilder() - .setUuid("b4e698d6-2dca-41ea-8483-263d40372071") + .setUuid(recordUuid) ) .setType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) .build() -- GitLab From 0b1c4f9396419299092cd7c133f584e421ff96e2 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 13 Jul 2022 12:09:57 +0200 Subject: [PATCH 12/36] Use randomized user name to connect blockchain --- src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt index 6be4219a7..0ff89ae3e 100644 --- a/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt +++ b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt @@ -41,13 +41,15 @@ import fabric.FabricConnector import io.grpc.Server import io.grpc.ServerBuilder import proto.Config +import kotlin.random.Random +import kotlin.random.nextUInt class FabricServer(val port: Int) { private val server: Server init { val cfg = Config.DltConfig.newBuilder().setWallet("wallet").setConnectionFile("config/connection-org1.json") - .setUser("appUser") + .setUser("appUser" + Random.nextUInt()) .setChannel("dlt") .setContract("basic").setCaCertFile("config/ca.org1.example.com-cert.pem").setCaUrl("https://teraflow.nlehd.de:7054") .setCaAdmin("admin").setCaAdminSecret("adminpw").setMsp("Org1MSP").setAffiliation("org1.department1") -- GitLab From 0c0a24c1f4e860e982b93c391e7c3a6fd47a6cb7 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 13 Jul 2022 15:41:13 +0200 Subject: [PATCH 13/36] DLT: Add support for updating and deleting records --- .../config/ca.org1.example.com-cert.pem | 16 ++++----- src/dlt/gateway/config/connection-org1.json | 8 ++--- src/dlt/gateway/src/main/kotlin/Main.kt | 31 ++++++++++++++++ .../src/main/kotlin/fabric/FabricConnector.kt | 12 +++++-- .../src/main/kotlin/grpc/GrpcHandler.kt | 36 ++++++++++--------- 5 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/dlt/gateway/config/ca.org1.example.com-cert.pem b/src/dlt/gateway/config/ca.org1.example.com-cert.pem index aca53bf6e..dd98ff366 100644 --- a/src/dlt/gateway/config/ca.org1.example.com-cert.pem +++ b/src/dlt/gateway/config/ca.org1.example.com-cert.pem @@ -1,14 +1,14 @@ -----BEGIN CERTIFICATE----- -MIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw +MIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw cDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH EwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh -Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw +Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw WjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV BAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT -Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u -lEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU -4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG -AQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD -AgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6 -H4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz +Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8 +zJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw +2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG +AQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD +AgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk +FizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80= -----END CERTIFICATE----- diff --git a/src/dlt/gateway/config/connection-org1.json b/src/dlt/gateway/config/connection-org1.json index 17170a215..5e15b1169 100644 --- a/src/dlt/gateway/config/connection-org1.json +++ b/src/dlt/gateway/config/connection-org1.json @@ -26,7 +26,7 @@ "peer0.org1.example.com": { "url": "grpcs://teraflow.nlehd.de:7051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u\nlEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU\n4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD\nAgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6\nH4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8\nzJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw\n2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD\nAgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk\nFizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80=\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org1.example.com", @@ -36,7 +36,7 @@ "peer0.org2.example.com": { "url": "grpcs://teraflow.nlehd.de:9051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUMyzCixoqXQBzqrCTv301KO5Ub3wwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTIwODQxMDBaFw0zNzA3MDgwODQxMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATpsE9qr9oDDxEZ\nLNDGZduPAslCFl3AY8wC2dURdTJQmn3V/AMeFGiC+a9xEXF5+C/UR/GHE8Cca8/j\nK8GtiM06o0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUE4+hU1/yQ/NCcqrhtAOcSj3O/3EwCgYIKoZIzj0EAwIDRwAwRAIg\nVDsbCz7hPyZ2tULxWiTUnG0aPg7TBh6lD5mRMObX1vUCIA1kcgFnfMPWA1lJJMRQ\nqGw7ydxYTyZ4X4M1b1PPAa3f\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICHzCCAcWgAwIBAgIUOsbw943VjFitBzBEXAcxLlOLow8wCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTMxNDUyMDBaFw0zNzA3MDkxNDUyMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARKov9QrWe9dklr\nqYJxOOoqoO3EI18LtPizVCKYrG3IMqkaatM6DLc/2SBTSIqIDeTR6N8WZAPVgLKA\nboRt87Heo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUYe4sT8J0Y2iN/9Q14FcpmxK5dYcwCgYIKoZIzj0EAwIDSAAwRQIh\nAJh2+1QXc5/ZlPaN2qnh3TXbzPFeCDXjoMAoBsOYtRcMAiAF+tcoFY0NcLDfTZBm\nNuuiQDIasoTB9Wuu4PBv8T7I5w==\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org2.example.com", @@ -50,7 +50,7 @@ "caName": "ca-org1", "tlsCACerts": { "pem": [ - "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUGgC/inWY+75+oERoZUbGo0YGtV4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEyMDg0MTAwWhcNMzcwNzA4MDg0MTAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL9u\nlEZaQqSTzwsf5zhWmPPytwiBrhma5wshEEDMeN2D49RgVJjtmSpImrZoucpgAzSU\n4ZvZkGQ4CznTp5kLUfSjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSRPP23/S9Hs3Fgddh4NIEhbQ9ExzAKBggqhkjOPQQD\nAgNIADBFAiEAv97c3x7fy5gfA+lmSb0XlWD+x1o7i09kGFHYDyFx8UkCIGVugTW6\nH4GmqRZJsVM1TbtbS3YUrZ4eVghstBqrhmBz\n-----END CERTIFICATE-----\n" + "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8\nzJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw\n2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD\nAgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk\nFizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80=\n-----END CERTIFICATE-----\n" ] }, "httpOptions": { @@ -62,7 +62,7 @@ "orderer0.example.com": { "url": "grpcs://teraflow.nlehd.de:7050", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIULAsUlO5yAbjj/6xnDbyiWeoj6rkwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxMjA4NDAwMFoXDTM3MDcwODA4NDAwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEjwa9/lOenYmcYL+Kqns6lV+UFt8+3tKZHlcwogZf\ngYnBEbsdL9oTZXSFgIvpCpQee65KKFUl7SqMDXrO3Mg0hqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFPuDBFVg+lhySQO0\nICEnNQ/TzRXeMAoGCCqGSM49BAMCA0cAMEQCIEadLxytDs79F5qTrTuI6SdaFoLB\n6AoQ5VGUNm2mF8tGAiAlWrLs8g3Ni2c7cjFeHZfiFy/2pqJLtor1ieGlFslq1w==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUA6RVRhhZD8pvcNAQJ6HWPUkbNNIwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxMzE0NTIwMFoXDTM3MDcwOTE0NTIwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEs7mmo5MY0oO9zCYcraUN/MXWxHiQ04xG6tWLaeWX\nqfSmFnfO8UBFq4OF5Li861UWGe+s9QxNXIELchBG8M7qLqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFBGG7kbCLEXzlQUF\nQfx845Qf2WKCMAoGCCqGSM49BAMCA0cAMEQCIAwABAqRopU+Qt3myyt4YiKQFpyO\nWTZifesVFe3fM/Q3AiA7oZ9XdQ5EuPhqFxSz7ASF45iyL3k078ilusjpc6IZ/g==\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "orderer0.example.com", diff --git a/src/dlt/gateway/src/main/kotlin/Main.kt b/src/dlt/gateway/src/main/kotlin/Main.kt index dc460b8fa..432965137 100644 --- a/src/dlt/gateway/src/main/kotlin/Main.kt +++ b/src/dlt/gateway/src/main/kotlin/Main.kt @@ -86,6 +86,8 @@ fun main() = runBlocking { val domainUuid = UUID.randomUUID().toString() val recordUuid = UUID.randomUUID().toString() + println("New domain uuid $domainUuid") + println("New record uuid $recordUuid") val id = Dlt.DltRecordId.newBuilder() .setDomainUuid( @@ -104,6 +106,7 @@ fun main() = runBlocking { .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_LINK) .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) + .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) .build() @@ -117,8 +120,36 @@ fun main() = runBlocking { .setDataJson("{}") .build() + println("sending new record") client.putData(data) client.getData(id) Thread.sleep(5000) + + val updateData = Dlt.DltRecord.newBuilder() + .setRecordId(id) + .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) + .setDataJson("{\"name\": \"test\"}") + .build() + + println("updating record") + client.putData(updateData) + client.getData(id) + + Thread.sleep(5000) + + val removeData = Dlt.DltRecord.newBuilder() + .setRecordId(id) + .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) + .setDataJson("{\"name\": \"test\"}") + .build() + + println("removing record") + client.putData(removeData) + try { + client.getData(id) + } catch (e: Exception) { + println(e.toString()) + } + Thread.sleep(5000) } diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index b6200f3f0..e4188d156 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -118,6 +118,7 @@ class FabricConnector(val config: Config.DltConfig) { return String( contract.submitTransaction( "UpdateRecord", + record.recordId.domainUuid.uuid, record.recordId.recordUuid.uuid, record.recordId.type.number.toString(), record.dataJson @@ -125,8 +126,15 @@ class FabricConnector(val config: Config.DltConfig) { ) } - fun deleteData(uuid: String): String { - return String(contract.submitTransaction("DeactivateRecord", uuid)) + fun deleteData(record: Dlt.DltRecord): String { + return String( + contract.submitTransaction( + "DeleteRecord", + record.recordId.domainUuid.uuid, + record.recordId.recordUuid.uuid, + record.recordId.type.number.toString() + ) + ) } fun subscribeForEvents(): Channel { diff --git a/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt index ae1b35faa..c3d6d801f 100644 --- a/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt +++ b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt @@ -40,20 +40,34 @@ package grpc import proto.Dlt import fabric.FabricConnector import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.consumeAsFlow import proto.ContextOuterClass import proto.DltServiceGrpcKt -class DLTService(private val connector: FabricConnector): DltServiceGrpcKt.DltServiceCoroutineImplBase() { +class DLTService(private val connector: FabricConnector) : DltServiceGrpcKt.DltServiceCoroutineImplBase() { override suspend fun recordToDlt(request: Dlt.DltRecord): Dlt.DltRecordStatus { println("Incoming request ${request.recordId.recordUuid}") - val error = connector.putData(request) + val error = when (request.operation) { + Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD -> { + println("Adding new record") + connector.putData(request) + } + Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE -> { + println("Updating record") + connector.updateData(request) + } + Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE -> { + println("Deleting record") + connector.deleteData(request) + } + else -> "Undefined or unknown operation" + } + val dltStatusEnum: Dlt.DltRecordStatusEnum = if (error == "") { Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_SUCCEEDED } else { Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_FAILED } - println("Put data status: $error") return Dlt.DltRecordStatus.newBuilder() .setRecordId(request.recordId) .setStatus(dltStatusEnum) @@ -65,19 +79,9 @@ class DLTService(private val connector: FabricConnector): DltServiceGrpcKt.DltSe return connector.getData(request.recordUuid.uuid) } - override fun subscribeToDlt(request: Dlt.DltRecordSubscription): Flow = flow { + override fun subscribeToDlt(request: Dlt.DltRecordSubscription): Flow { println("Subscription request: $request") - val c = connector.subscribeForEvents() - println("Got channel") - val event = c.receive() - println("Received event from channel") - - // TODO: mistype, proto scheme fix required - if (request.operationValueList.contains(event.event.eventType.number)) { - if (request.typeList.contains(event.recordId.type)) { - emit(event) - } - } + return connector.subscribeForEvents().consumeAsFlow() } override suspend fun getDltStatus(request: ContextOuterClass.TeraFlowController): Dlt.DltPeerStatus { -- GitLab From 25668e43bd8c6360d5c6a34f2c3499749bb7f0c3 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Mon, 18 Jul 2022 18:15:04 +0200 Subject: [PATCH 14/36] Update readme file for DLT gateway service --- src/dlt/gateway/README.md | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/dlt/gateway/README.md b/src/dlt/gateway/README.md index 361de07c6..505b1c135 100644 --- a/src/dlt/gateway/README.md +++ b/src/dlt/gateway/README.md @@ -13,7 +13,7 @@ duplication of the object or source code - either totally or in part - is strictly prohibited. - Copyright (c) 2021 NEC Laboratories Europe GmbH + Copyright (c) 2022 NEC Laboratories Europe GmbH All Rights Reserved. Authors: Konstantin Munichev @@ -42,7 +42,7 @@ ## General information The DLT module is used to provide access to the underlying Fabric deployment. It allows clients to add, retrieve, modify and delete blockchain-backed data, essentially working as a key-value -database. External clients should use REST API to communicate with this service, its detailed +database. External clients should use gRPC API to communicate with this service, its detailed description available below. ## Code structure @@ -59,26 +59,17 @@ CRUD interface. Other files contain auxiliary code for `FabricConnector` which allows it to register/enroll users and to obtain smart contract instances. -### HTTP package -Contains server side HTTP handler. It accepts requests from the outside and performs the -requested operation. For the detailed description see API description section. +### Grpc package +Contains server side gRPC handler. It accepts requests from the outside and performs the +requested operation. For the more detailed description see Proto package description right below. ### Proto package -The proto package contains `Config.proto` file which contains messages for REST API. The most -important ones are `DltConfig` (it defines the whole DLT configuration) and `DltRecord` which -represents data to store in the blockchain. +The proto package contains `dlt.proto` file which defines gRPC service `DltService` API and messages +it uses. There are 3 main functions: `RecordToDlt` which allows to create/modify/delete data, +`GetFromDlt` which returns already written data and `SubscribeToDlt` which allows clients subscribe +for future create/modify/delete events with provided filters. +Other proto files don't play any significant role and could be safely ignored by end users. ### Client example This code is not necessary to the service, but it could be used to test the service. It contains -a sample REST client which connects the service and perform all the CRUD operations. - -## REST API description -| Method | URL | Input | Response code | Output | -| --- | ----------- | --- | --- | --- | -| POST | /dlt/configure | Configuration object | 201 or 400 | Status value | -| GET | /dlt/configure | - | 200 or 404 | Configuration object | -| POST | /dlt/record | Record object | 200, 201, 400 or 404 | Status value | -| GET | /dlt/record | Record id | 200 or 404 | Record object | - -Record and configuration object are defined in `proto` package. - +a sample gRPC client which connects the service and perform all the CRUD operations. -- GitLab From 88be13343490a0ab11a3bc2f3eb77cb118b04d04 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 19 Jul 2022 15:27:46 +0200 Subject: [PATCH 15/36] Update cerificate and connection files --- .../gateway/config/ca.org1.example.com-cert.pem | 16 ++++++++-------- src/dlt/gateway/config/connection-org1.json | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/dlt/gateway/config/ca.org1.example.com-cert.pem b/src/dlt/gateway/config/ca.org1.example.com-cert.pem index dd98ff366..442983f12 100644 --- a/src/dlt/gateway/config/ca.org1.example.com-cert.pem +++ b/src/dlt/gateway/config/ca.org1.example.com-cert.pem @@ -1,14 +1,14 @@ -----BEGIN CERTIFICATE----- -MIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw +MIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw cDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH EwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh -Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw +Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw WjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV BAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT -Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8 -zJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw -2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG -AQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD -AgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk -FizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80= +Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk ++U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO +PXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG +AQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD +AgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp +rPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k= -----END CERTIFICATE----- diff --git a/src/dlt/gateway/config/connection-org1.json b/src/dlt/gateway/config/connection-org1.json index 5e15b1169..068404e7b 100644 --- a/src/dlt/gateway/config/connection-org1.json +++ b/src/dlt/gateway/config/connection-org1.json @@ -26,7 +26,7 @@ "peer0.org1.example.com": { "url": "grpcs://teraflow.nlehd.de:7051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8\nzJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw\n2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD\nAgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk\nFizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80=\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk\n+U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO\nPXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD\nAgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp\nrPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k=\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org1.example.com", @@ -36,7 +36,7 @@ "peer0.org2.example.com": { "url": "grpcs://teraflow.nlehd.de:9051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICHzCCAcWgAwIBAgIUOsbw943VjFitBzBEXAcxLlOLow8wCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTMxNDUyMDBaFw0zNzA3MDkxNDUyMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARKov9QrWe9dklr\nqYJxOOoqoO3EI18LtPizVCKYrG3IMqkaatM6DLc/2SBTSIqIDeTR6N8WZAPVgLKA\nboRt87Heo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUYe4sT8J0Y2iN/9Q14FcpmxK5dYcwCgYIKoZIzj0EAwIDSAAwRQIh\nAJh2+1QXc5/ZlPaN2qnh3TXbzPFeCDXjoMAoBsOYtRcMAiAF+tcoFY0NcLDfTZBm\nNuuiQDIasoTB9Wuu4PBv8T7I5w==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUfNAV+OorBxKgF5kz/zfKH9s8vsowCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTkxMzAwMDBaFw0zNzA3MTUxMzAwMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQo2aut3d3aKCLs\n6jf+GBOQ+LoTJG1WtTJXCYfwz3UeBqnDPs6XdADQ+tPAB3xLdAqbDLAdniTdJ5uI\n7iD5ujeZo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUb56rcdREBjw0Y2DFn1xrwOnIroUwCgYIKoZIzj0EAwIDRwAwRAIg\nONguVx49qjxsiw9Bdpqmqopd7nV6NczIBAtt/9U2mfcCIEePn5jGZaoJVPpkgN1D\nXgy1XywDqFL05nY/gDnDTJFo\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org2.example.com", @@ -50,7 +50,7 @@ "caName": "ca-org1", "tlsCACerts": { "pem": [ - "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUVSFCRQNScNJyIwJz89vi6tazYJ4wCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzEzMTQ1MjAwWhcNMzcwNzA5MTQ1MjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLg8\nzJvOKBR1fS+hrrMc3xEhOalkSG+5vGGpwzgHv5NNMlDZcbQVuEBHPmRowbLWx4nw\n2CIUSfkhtQGC5GmUncOjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRm7lBNvvkP8iy6f0cXwz9BTeSYDzAKBggqhkjOPQQD\nAgNHADBEAiB1p8QFXHszJsMqiShmyuTA6zGPfHlnan3dgYMCTNR7mwIgFx3SqSYk\nFizTEfVfOcdjIYGc3pTPTyK5PVbO1k2BW80=\n-----END CERTIFICATE-----\n" + "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk\n+U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO\nPXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD\nAgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp\nrPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k=\n-----END CERTIFICATE-----\n" ] }, "httpOptions": { @@ -62,7 +62,7 @@ "orderer0.example.com": { "url": "grpcs://teraflow.nlehd.de:7050", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUA6RVRhhZD8pvcNAQJ6HWPUkbNNIwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxMzE0NTIwMFoXDTM3MDcwOTE0NTIwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEs7mmo5MY0oO9zCYcraUN/MXWxHiQ04xG6tWLaeWX\nqfSmFnfO8UBFq4OF5Li861UWGe+s9QxNXIELchBG8M7qLqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFBGG7kbCLEXzlQUF\nQfx845Qf2WKCMAoGCCqGSM49BAMCA0cAMEQCIAwABAqRopU+Qt3myyt4YiKQFpyO\nWTZifesVFe3fM/Q3AiA7oZ9XdQ5EuPhqFxSz7ASF45iyL3k078ilusjpc6IZ/g==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUAnklc7GR28OpDSqO+beAokDGNjUwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxOTEzMDAwMFoXDTM3MDcxNTEzMDAwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEfVGvu1yLzzOuv0xqdLozKOrGIEHjmv0UZ1/IEznG\nbxygeZDsmsagQEJFRPSG9SnrE9XL4qnwRYHAEOyi5hVHGqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFIjQ/caW9qF+Db2J\nxOtvNvlg2nnxMAoGCCqGSM49BAMCA0cAMEQCIH3rEsvHrUrrEgnVvJmo7xJg5MaR\nvhKqcCC0oNglr8CNAiB6iXUVuXirVF7z3QaUp6DSxuNEHoH6P916pjhEs2U5oA==\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "orderer0.example.com", -- GitLab From 58fd9bb7046a0578f33e62707db52415abf67c23 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Tue, 19 Jul 2022 14:01:32 +0000 Subject: [PATCH 16/36] DLT component: - added simple test script --- src/dlt/connector/main_test.py | 42 ++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/dlt/connector/main_test.py b/src/dlt/connector/main_test.py index a5ad9c33c..632497220 100644 --- a/src/dlt/connector/main_test.py +++ b/src/dlt/connector/main_test.py @@ -1,15 +1,43 @@ -import sys +# pip install grpcio==1.47.0 grpcio-tools==1.47.0 protobuf==3.20.1 +# PYTHONPATH=/home/cttc/teraflow/src python -m dlt.connector.main_test + +import logging, sys, time +from common.proto.dlt_gateway_pb2 import DLTRECORDOPERATION_ADD, DLTRECORDTYPE_DEVICE, DltRecord +from common.tools.object_factory.Device import json_device +from common.tools.grpc.Tools import grpc_message_to_json_string +from src.common.proto.context_pb2 import DEVICEOPERATIONALSTATUS_ENABLED, Device from .client.DltGatewayClient import DltGatewayClient from .client.DltEventsCollector import DltEventsCollector +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + def main(): - dltgateway_client_1 = DltGatewayClient(host='', port=0) - dltgateway_client_2 = DltGatewayClient(host='', port=0) - dltgateway_client_3 = DltGatewayClient(host='', port=0) + dltgateway_client = DltGatewayClient(host='127.0.0.1', port=50051) + dltgateway_collector = DltEventsCollector(dltgateway_client, log_events_received=True) + dltgateway_collector.start() + + time.sleep(3) + + device = Device(**json_device('dev-1', 'packet-router', DEVICEOPERATIONALSTATUS_ENABLED)) + + r2dlt_req = DltRecord() + r2dlt_req.record_id.domain_uuid.uuid = 'tfs-a' + r2dlt_req.record_id.type = DLTRECORDTYPE_DEVICE + r2dlt_req.record_id.record_uuid.uuid = device.device_id.device_uuid.uuid + r2dlt_req.operation = DLTRECORDOPERATION_ADD + r2dlt_req.data_json = grpc_message_to_json_string(device) + LOGGER.info('r2dlt_req = {:s}'.format(grpc_message_to_json_string(r2dlt_req))) + r2dlt_rep = dltgateway_client.RecordToDlt(r2dlt_req) + LOGGER.info('r2dlt_rep = {:s}'.format(grpc_message_to_json_string(r2dlt_rep))) + + dlt2r_req = r2dlt_req.record_id + LOGGER.info('dlt2r_req = {:s}'.format(grpc_message_to_json_string(dlt2r_req))) + dlt2r_rep = dltgateway_client.RecordToDlt(dlt2r_req) + LOGGER.info('dlt2r_rep = {:s}'.format(grpc_message_to_json_string(dlt2r_rep))) - dltgateway_collector_1 = DltEventsCollector(dltgateway_client_1, log_events_received=True) - dltgateway_collector_2 = DltEventsCollector(dltgateway_client_2, log_events_received=True) - dltgateway_collector_3 = DltEventsCollector(dltgateway_client_3, log_events_received=True) + dltgateway_collector.stop() + return 0 if __name__ == '__main__': sys.exit(main()) -- GitLab From 3f18c2ac4d90441824d79b6b15a3ccd7048cf47f Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 19 Jul 2022 16:32:16 +0200 Subject: [PATCH 17/36] Use proto files from the top root of the project --- src/dlt/gateway/build.gradle.kts | 1 + src/dlt/gateway/src/main/kotlin/Main.kt | 45 +-- .../src/main/kotlin/fabric/FabricConnector.kt | 20 +- .../src/main/kotlin/grpc/GrpcHandler.kt | 33 +- .../src/main/kotlin/proto/context.proto | 374 ------------------ .../gateway/src/main/kotlin/proto/dlt.proto | 97 ----- .../main/kotlin/proto/kpi_sample_types.proto | 24 -- 7 files changed, 51 insertions(+), 543 deletions(-) delete mode 100644 src/dlt/gateway/src/main/kotlin/proto/context.proto delete mode 100644 src/dlt/gateway/src/main/kotlin/proto/dlt.proto delete mode 100644 src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto diff --git a/src/dlt/gateway/build.gradle.kts b/src/dlt/gateway/build.gradle.kts index 92d0b6bc7..c7213fbf7 100644 --- a/src/dlt/gateway/build.gradle.kts +++ b/src/dlt/gateway/build.gradle.kts @@ -107,6 +107,7 @@ task("runServer", JavaExec::class) { sourceSets { main { proto { + srcDir("../../../proto") srcDir("src/main/kotlin/proto") } } diff --git a/src/dlt/gateway/src/main/kotlin/Main.kt b/src/dlt/gateway/src/main/kotlin/Main.kt index 432965137..576aeee56 100644 --- a/src/dlt/gateway/src/main/kotlin/Main.kt +++ b/src/dlt/gateway/src/main/kotlin/Main.kt @@ -35,34 +35,35 @@ // // THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. +import context.ContextOuterClass import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking -import proto.ContextOuterClass -import proto.Dlt -import proto.DltServiceGrpcKt +import dlt.DltGateway +import dlt.DltGatewayServiceGrpcKt import java.io.Closeable import java.util.* import java.util.concurrent.TimeUnit class DltServiceClient(private val channel: ManagedChannel) : Closeable { - private val stub: DltServiceGrpcKt.DltServiceCoroutineStub = DltServiceGrpcKt.DltServiceCoroutineStub(channel) + private val stub: DltGatewayServiceGrpcKt.DltGatewayServiceCoroutineStub = + DltGatewayServiceGrpcKt.DltGatewayServiceCoroutineStub(channel) - suspend fun putData(data: Dlt.DltRecord) { + suspend fun putData(data: DltGateway.DltRecord) { println("Sending record ${data.recordId}...") val response = stub.recordToDlt(data) println("Response: ${response.recordId}") } - suspend fun getData(id: Dlt.DltRecordId) { + suspend fun getData(id: DltGateway.DltRecordId) { println("Requesting record $id...") val response = stub.getFromDlt(id) println("Got data: $response") } - fun subscribe(filter: Dlt.DltRecordSubscription) { + fun subscribe(filter: DltGateway.DltRecordSubscription) { val subscription = stub.subscribeToDlt(filter) GlobalScope.launch { subscription.collect { @@ -89,7 +90,7 @@ fun main() = runBlocking { println("New domain uuid $domainUuid") println("New record uuid $recordUuid") - val id = Dlt.DltRecordId.newBuilder() + val id = DltGateway.DltRecordId.newBuilder() .setDomainUuid( ContextOuterClass.Uuid.newBuilder() .setUuid(domainUuid) @@ -98,25 +99,25 @@ fun main() = runBlocking { ContextOuterClass.Uuid.newBuilder() .setUuid(recordUuid) ) - .setType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) + .setType(DltGateway.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) .build() - val subscription = Dlt.DltRecordSubscription.newBuilder() - .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_CONTEXT) - .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_LINK) - .addType(Dlt.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) - .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) - .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) - .addOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) + val subscription = DltGateway.DltRecordSubscription.newBuilder() + .addType(DltGateway.DltRecordTypeEnum.DLTRECORDTYPE_CONTEXT) + .addType(DltGateway.DltRecordTypeEnum.DLTRECORDTYPE_LINK) + .addType(DltGateway.DltRecordTypeEnum.DLTRECORDTYPE_SERVICE) + .addOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) + .addOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) + .addOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) .build() client.subscribe(subscription) Thread.sleep(5000) - val data = Dlt.DltRecord.newBuilder() + val data = DltGateway.DltRecord.newBuilder() .setRecordId(id) - .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) + .setOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) .setDataJson("{}") .build() @@ -126,9 +127,9 @@ fun main() = runBlocking { Thread.sleep(5000) - val updateData = Dlt.DltRecord.newBuilder() + val updateData = DltGateway.DltRecord.newBuilder() .setRecordId(id) - .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) + .setOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE) .setDataJson("{\"name\": \"test\"}") .build() @@ -138,9 +139,9 @@ fun main() = runBlocking { Thread.sleep(5000) - val removeData = Dlt.DltRecord.newBuilder() + val removeData = DltGateway.DltRecord.newBuilder() .setRecordId(id) - .setOperation(Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) + .setOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE) .setDataJson("{\"name\": \"test\"}") .build() diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index e4188d156..d175f6011 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -37,9 +37,9 @@ package fabric +import dlt.DltGateway import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.runBlocking -import proto.Dlt import org.hyperledger.fabric.gateway.Contract import org.hyperledger.fabric.gateway.ContractEvent import org.hyperledger.fabric.gateway.Wallet @@ -56,7 +56,7 @@ class FabricConnector(val config: Config.DltConfig) { private val wallet: Wallet private val contract: Contract - private val channels: MutableList> = mutableListOf() + private val channels: MutableList> = mutableListOf() init { // Create a CA client for interacting with the CA. @@ -78,7 +78,7 @@ class FabricConnector(val config: Config.DltConfig) { val consumer = Consumer { event: ContractEvent? -> run { println("new event detected") - val parsedEvent = Dlt.DltRecordEvent.parseFrom(event?.payload?.get()) + val parsedEvent = DltGateway.DltRecordEvent.parseFrom(event?.payload?.get()) println(parsedEvent.recordId.recordUuid) runBlocking { channels.forEach { @@ -96,7 +96,7 @@ class FabricConnector(val config: Config.DltConfig) { return getContract(config, wallet) } - fun putData(record: Dlt.DltRecord): String { + fun putData(record: DltGateway.DltRecord): String { println(record.toString()) return String( contract.submitTransaction( @@ -109,12 +109,12 @@ class FabricConnector(val config: Config.DltConfig) { ) } - fun getData(uuid: String): Dlt.DltRecord { + fun getData(uuid: String): DltGateway.DltRecord { val result = contract.evaluateTransaction("GetRecord", uuid) - return Dlt.DltRecord.parseFrom(result) + return DltGateway.DltRecord.parseFrom(result) } - fun updateData(record: Dlt.DltRecord): String { + fun updateData(record: DltGateway.DltRecord): String { return String( contract.submitTransaction( "UpdateRecord", @@ -126,7 +126,7 @@ class FabricConnector(val config: Config.DltConfig) { ) } - fun deleteData(record: Dlt.DltRecord): String { + fun deleteData(record: DltGateway.DltRecord): String { return String( contract.submitTransaction( "DeleteRecord", @@ -137,8 +137,8 @@ class FabricConnector(val config: Config.DltConfig) { ) } - fun subscribeForEvents(): Channel { - val produceCh = Channel() + fun subscribeForEvents(): Channel { + val produceCh = Channel() channels.add(produceCh) return produceCh } diff --git a/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt index c3d6d801f..d39c24a1a 100644 --- a/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt +++ b/src/dlt/gateway/src/main/kotlin/grpc/GrpcHandler.kt @@ -37,58 +37,59 @@ package grpc -import proto.Dlt import fabric.FabricConnector import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.consumeAsFlow -import proto.ContextOuterClass -import proto.DltServiceGrpcKt +import context.ContextOuterClass +import dlt.DltGateway +import dlt.DltGatewayServiceGrpcKt -class DLTService(private val connector: FabricConnector) : DltServiceGrpcKt.DltServiceCoroutineImplBase() { - override suspend fun recordToDlt(request: Dlt.DltRecord): Dlt.DltRecordStatus { +class DLTService(private val connector: FabricConnector) : + DltGatewayServiceGrpcKt.DltGatewayServiceCoroutineImplBase() { + override suspend fun recordToDlt(request: DltGateway.DltRecord): DltGateway.DltRecordStatus { println("Incoming request ${request.recordId.recordUuid}") val error = when (request.operation) { - Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_ADD -> { + DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_ADD -> { println("Adding new record") connector.putData(request) } - Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE -> { + DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_UPDATE -> { println("Updating record") connector.updateData(request) } - Dlt.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE -> { + DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_DELETE -> { println("Deleting record") connector.deleteData(request) } else -> "Undefined or unknown operation" } - val dltStatusEnum: Dlt.DltRecordStatusEnum = if (error == "") { - Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_SUCCEEDED + val dltStatusEnum: DltGateway.DltRecordStatusEnum = if (error == "") { + DltGateway.DltRecordStatusEnum.DLTRECORDSTATUS_SUCCEEDED } else { - Dlt.DltRecordStatusEnum.DLTRECORDSTATUS_FAILED + DltGateway.DltRecordStatusEnum.DLTRECORDSTATUS_FAILED } - return Dlt.DltRecordStatus.newBuilder() + return DltGateway.DltRecordStatus.newBuilder() .setRecordId(request.recordId) .setStatus(dltStatusEnum) .setErrorMessage(error) .build() } - override suspend fun getFromDlt(request: Dlt.DltRecordId): Dlt.DltRecord { + override suspend fun getFromDlt(request: DltGateway.DltRecordId): DltGateway.DltRecord { return connector.getData(request.recordUuid.uuid) } - override fun subscribeToDlt(request: Dlt.DltRecordSubscription): Flow { + override fun subscribeToDlt(request: DltGateway.DltRecordSubscription): Flow { println("Subscription request: $request") return connector.subscribeForEvents().consumeAsFlow() } - override suspend fun getDltStatus(request: ContextOuterClass.TeraFlowController): Dlt.DltPeerStatus { + override suspend fun getDltStatus(request: ContextOuterClass.TeraFlowController): DltGateway.DltPeerStatus { return super.getDltStatus(request) } - override suspend fun getDltPeers(request: ContextOuterClass.Empty): Dlt.DltPeerStatusList { + override suspend fun getDltPeers(request: ContextOuterClass.Empty): DltGateway.DltPeerStatusList { return super.getDltPeers(request) } } \ No newline at end of file diff --git a/src/dlt/gateway/src/main/kotlin/proto/context.proto b/src/dlt/gateway/src/main/kotlin/proto/context.proto deleted file mode 100644 index a7185ba10..000000000 --- a/src/dlt/gateway/src/main/kotlin/proto/context.proto +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package proto; - -import "kpi_sample_types.proto"; - -service ContextService { - rpc ListContextIds (Empty ) returns ( ContextIdList ) {} - rpc ListContexts (Empty ) returns ( ContextList ) {} - rpc GetContext (ContextId ) returns ( Context ) {} - rpc SetContext (Context ) returns ( ContextId ) {} - rpc RemoveContext (ContextId ) returns ( Empty ) {} - rpc GetContextEvents (Empty ) returns (stream ContextEvent ) {} - - rpc ListTopologyIds (ContextId ) returns ( TopologyIdList ) {} - rpc ListTopologies (ContextId ) returns ( TopologyList ) {} - rpc GetTopology (TopologyId ) returns ( Topology ) {} - rpc SetTopology (Topology ) returns ( TopologyId ) {} - rpc RemoveTopology (TopologyId ) returns ( Empty ) {} - rpc GetTopologyEvents (Empty ) returns (stream TopologyEvent ) {} - - rpc ListDeviceIds (Empty ) returns ( DeviceIdList ) {} - rpc ListDevices (Empty ) returns ( DeviceList ) {} - rpc GetDevice (DeviceId ) returns ( Device ) {} - rpc SetDevice (Device ) returns ( DeviceId ) {} - rpc RemoveDevice (DeviceId ) returns ( Empty ) {} - rpc GetDeviceEvents (Empty ) returns (stream DeviceEvent ) {} - - rpc ListLinkIds (Empty ) returns ( LinkIdList ) {} - rpc ListLinks (Empty ) returns ( LinkList ) {} - rpc GetLink (LinkId ) returns ( Link ) {} - rpc SetLink (Link ) returns ( LinkId ) {} - rpc RemoveLink (LinkId ) returns ( Empty ) {} - rpc GetLinkEvents (Empty ) returns (stream LinkEvent ) {} - - rpc ListServiceIds (ContextId ) returns ( ServiceIdList ) {} - rpc ListServices (ContextId ) returns ( ServiceList ) {} - rpc GetService (ServiceId ) returns ( Service ) {} - rpc SetService (Service ) returns ( ServiceId ) {} - rpc RemoveService (ServiceId ) returns ( Empty ) {} - rpc GetServiceEvents (Empty ) returns (stream ServiceEvent ) {} - - rpc ListSliceIds (ContextId ) returns ( SliceIdList ) {} - rpc ListSlices (ContextId ) returns ( SliceList ) {} - rpc GetSlice (SliceId ) returns ( Slice ) {} - rpc SetSlice (Slice ) returns ( SliceId ) {} - rpc RemoveSlice (SliceId ) returns ( Empty ) {} - rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} - - rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} - rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} - rpc GetConnection (ConnectionId) returns ( Connection ) {} - rpc SetConnection (Connection ) returns ( ConnectionId ) {} - rpc RemoveConnection (ConnectionId) returns ( Empty ) {} - rpc GetConnectionEvents(Empty ) returns (stream ConnectionEvent ) {} -} - -// ----- Generic ------------------------------------------------------------------------------------------------------- -message Empty {} - -message Uuid { - string uuid = 1; -} - -enum EventTypeEnum { - EVENTTYPE_UNDEFINED = 0; - EVENTTYPE_CREATE = 1; - EVENTTYPE_UPDATE = 2; - EVENTTYPE_REMOVE = 3; -} - -message Event { - double timestamp = 1; - EventTypeEnum event_type = 2; -} - -// ----- Context ------------------------------------------------------------------------------------------------------- -message ContextId { - Uuid context_uuid = 1; -} - -message Context { - ContextId context_id = 1; - repeated TopologyId topology_ids = 2; - repeated ServiceId service_ids = 3; - TeraFlowController controller = 4; -} - -message ContextIdList { - repeated ContextId context_ids = 1; -} - -message ContextList { - repeated Context contexts = 1; -} - -message ContextEvent { - Event event = 1; - ContextId context_id = 2; -} - - -// ----- Topology ------------------------------------------------------------------------------------------------------ -message TopologyId { - ContextId context_id = 1; - Uuid topology_uuid = 2; -} - -message Topology { - TopologyId topology_id = 1; - repeated DeviceId device_ids = 2; - repeated LinkId link_ids = 3; -} - -message TopologyIdList { - repeated TopologyId topology_ids = 1; -} - -message TopologyList { - repeated Topology topologies = 1; -} - -message TopologyEvent { - Event event = 1; - TopologyId topology_id = 2; -} - - -// ----- Device -------------------------------------------------------------------------------------------------------- -message DeviceId { - Uuid device_uuid = 1; -} - -message Device { - DeviceId device_id = 1; - string device_type = 2; - DeviceConfig device_config = 3; - DeviceOperationalStatusEnum device_operational_status = 4; - repeated DeviceDriverEnum device_drivers = 5; - repeated EndPoint device_endpoints = 6; -} - -message DeviceConfig { - repeated ConfigRule config_rules = 1; -} - -enum DeviceDriverEnum { - DEVICEDRIVER_UNDEFINED = 0; // also used for emulated - DEVICEDRIVER_OPENCONFIG = 1; - DEVICEDRIVER_TRANSPORT_API = 2; - DEVICEDRIVER_P4 = 3; - DEVICEDRIVER_IETF_NETWORK_TOPOLOGY = 4; - DEVICEDRIVER_ONF_TR_352 = 5; -} - -enum DeviceOperationalStatusEnum { - DEVICEOPERATIONALSTATUS_UNDEFINED = 0; - DEVICEOPERATIONALSTATUS_DISABLED = 1; - DEVICEOPERATIONALSTATUS_ENABLED = 2; -} - -message DeviceIdList { - repeated DeviceId device_ids = 1; -} - -message DeviceList { - repeated Device devices = 1; -} - -message DeviceEvent { - Event event = 1; - DeviceId device_id = 2; -} - - -// ----- Link ---------------------------------------------------------------------------------------------------------- -message LinkId { - Uuid link_uuid = 1; -} - -message Link { - LinkId link_id = 1; - repeated EndPointId link_endpoint_ids = 2; -} - -message LinkIdList { - repeated LinkId link_ids = 1; -} - -message LinkList { - repeated Link links = 1; -} - -message LinkEvent { - Event event = 1; - LinkId link_id = 2; -} - - -// ----- Service ------------------------------------------------------------------------------------------------------- -message ServiceId { - ContextId context_id = 1; - Uuid service_uuid = 2; -} - -message Service { - ServiceId service_id = 1; - ServiceTypeEnum service_type = 2; - repeated EndPointId service_endpoint_ids = 3; - repeated Constraint service_constraints = 4; - ServiceStatus service_status = 5; - ServiceConfig service_config = 6; -} - -enum ServiceTypeEnum { - SERVICETYPE_UNKNOWN = 0; - SERVICETYPE_L3NM = 1; - SERVICETYPE_L2NM = 2; - SERVICETYPE_TAPI_CONNECTIVITY_SERVICE = 3; -} - -enum ServiceStatusEnum { - SERVICESTATUS_UNDEFINED = 0; - SERVICESTATUS_PLANNED = 1; - SERVICESTATUS_ACTIVE = 2; - SERVICESTATUS_PENDING_REMOVAL = 3; -} - -message ServiceStatus { - ServiceStatusEnum service_status = 1; -} - -message ServiceConfig { - repeated ConfigRule config_rules = 1; -} - -message ServiceIdList { - repeated ServiceId service_ids = 1; -} - -message ServiceList { - repeated Service services = 1; -} - -message ServiceEvent { - Event event = 1; - ServiceId service_id = 2; -} - -// ----- Slice --------------------------------------------------------------------------------------------------------- -message SliceId { - ContextId context_id = 1; - Uuid slice_uuid = 2; -} - -message Slice { - SliceId slice_id = 1; - repeated EndPointId slice_endpoint_ids = 2; - repeated Constraint slice_constraints = 3; - repeated ServiceId slice_service_ids = 4; - repeated SliceId slice_subslice_ids = 5; - SliceStatus slice_status = 6; -} - -enum SliceStatusEnum { - SLICESTATUS_UNDEFINED = 0; - SLICESTATUS_PLANNED = 1; - SLICESTATUS_INIT = 2; - SLICESTATUS_ACTIVE = 3; - SLICESTATUS_DEINIT = 4; -} - -message SliceStatus { - SliceStatusEnum slice_status = 1; -} - -message SliceIdList { - repeated SliceId slice_ids = 1; -} - -message SliceList { - repeated Slice slices = 1; -} - -message SliceEvent { - Event event = 1; - SliceId slice_id = 2; -} - -// ----- Connection ---------------------------------------------------------------------------------------------------- -message ConnectionId { - Uuid connection_uuid = 1; -} - -message Connection { - ConnectionId connection_id = 1; - ServiceId service_id = 2; - repeated EndPointId path_hops_endpoint_ids = 3; - repeated ServiceId sub_service_ids = 4; -} - -message ConnectionIdList { - repeated ConnectionId connection_ids = 1; -} - -message ConnectionList { - repeated Connection connections = 1; -} - -message ConnectionEvent { - Event event = 1; - ConnectionId connection_id = 2; -} - - -// ----- Endpoint ------------------------------------------------------------------------------------------------------ -message EndPointId { - TopologyId topology_id = 1; - DeviceId device_id = 2; - Uuid endpoint_uuid = 3; -} - -message EndPoint { - EndPointId endpoint_id = 1; - string endpoint_type = 2; - repeated KpiSampleType kpi_sample_types = 3; -} - - -// ----- Configuration ------------------------------------------------------------------------------------------------- -enum ConfigActionEnum { - CONFIGACTION_UNDEFINED = 0; - CONFIGACTION_SET = 1; - CONFIGACTION_DELETE = 2; -} - -message ConfigRule { - ConfigActionEnum action = 1; - string resource_key = 2; - string resource_value = 3; -} - - -// ----- Constraint ---------------------------------------------------------------------------------------------------- -message Constraint { - string constraint_type = 1; - string constraint_value = 2; -} - - -// ----- Miscellaneous ------------------------------------------------------------------------------------------------- -message TeraFlowController { - ContextId context_id = 1; - string ip_address = 2; - uint32 port = 3; -} - -message AuthenticationResult { - ContextId context_id = 1; - bool authenticated = 2; -} diff --git a/src/dlt/gateway/src/main/kotlin/proto/dlt.proto b/src/dlt/gateway/src/main/kotlin/proto/dlt.proto deleted file mode 100644 index 7b145d2bc..000000000 --- a/src/dlt/gateway/src/main/kotlin/proto/dlt.proto +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package proto; - -import "context.proto"; - -service DltService { - rpc RecordToDlt (DltRecord ) returns ( DltRecordStatus ) {} - rpc GetFromDlt (DltRecordId ) returns ( DltRecord ) {} - rpc SubscribeToDlt(DltRecordSubscription ) returns (stream DltRecordEvent ) {} - rpc GetDltStatus (TeraFlowController ) returns ( DltPeerStatus ) {} // NEC is checking if it is possible - rpc GetDltPeers (Empty ) returns ( DltPeerStatusList) {} // NEC is checking if it is possible -} - -enum DltRecordTypeEnum { - DLTRECORDTYPE_UNDEFINED = 0; - DLTRECORDTYPE_CONTEXT = 1; - DLTRECORDTYPE_TOPOLOGY = 2; - DLTRECORDTYPE_DEVICE = 3; - DLTRECORDTYPE_LINK = 4; - DLTRECORDTYPE_SERVICE = 5; - DLTRECORDTYPE_SLICE = 6; -} - -enum DltRecordOperationEnum { - DLTRECORDOPERATION_UNDEFINED = 0; - DLTRECORDOPERATION_ADD = 1; - DLTRECORDOPERATION_UPDATE = 2; - DLTRECORDOPERATION_DELETE = 3; -} - -enum DltRecordStatusEnum { - DLTRECORDSTATUS_UNDEFINED = 0; - DLTRECORDSTATUS_SUCCEEDED = 1; - DLTRECORDSTATUS_FAILED = 2; -} - -enum DltStatusEnum { - DLTSTATUS_UNDEFINED = 0; - DLTSTATUS_NOTAVAILABLE = 1; - DLTSTATUS_INITIALIZED = 2; - DLTSTATUS_AVAILABLE = 3; - DLTSTATUS_DEINIT = 4; -} - -message DltRecordId { - Uuid domain_uuid = 1; // unique identifier of domain owning the record - DltRecordTypeEnum type = 2; // type of record - Uuid record_uuid = 3; // unique identifier of the record within the domain context_uuid/topology_uuid -} - -message DltRecord { - DltRecordId record_id = 1; // record identifier - DltRecordOperationEnum operation = 2; // operation to be performed over the record - string data_json = 3; // record content: JSON-encoded record content -} - -message DltRecordSubscription { - // retrieved events have to match ALL conditions. - // i.e., type in types requested, AND operation in operations requested - // TODO: consider adding a more sophisticated filtering - repeated DltRecordTypeEnum type = 1; // selected event types, empty=all - repeated DltRecordOperationEnum operation = 2; // selected event operations, empty=all -} - -message DltRecordEvent { - Event event = 1; // common event data (timestamp & event_type) - DltRecordId record_id = 2; // record identifier associated with this event -} - -message DltRecordStatus { - DltRecordId record_id = 1; // identifier of the associated record - DltRecordStatusEnum status = 2; // status of the record - string error_message = 3; // error message in case of failure, empty otherwise -} - -message DltPeerStatus { - TeraFlowController controller = 1; // Identifier of the TeraFlow controller instance - DltStatusEnum status = 2; // Status of the TeraFlow controller instance -} - -message DltPeerStatusList { - repeated DltPeerStatus peers = 1; // List of peers and their status -} diff --git a/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto b/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto deleted file mode 100644 index 82ef51e80..000000000 --- a/src/dlt/gateway/src/main/kotlin/proto/kpi_sample_types.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package proto; - -enum KpiSampleType { - KPISAMPLETYPE_UNKNOWN = 0; - KPISAMPLETYPE_PACKETS_TRANSMITTED = 101; - KPISAMPLETYPE_PACKETS_RECEIVED = 102; - KPISAMPLETYPE_BYTES_TRANSMITTED = 201; - KPISAMPLETYPE_BYTES_RECEIVED = 202; -} -- GitLab From 90e5ecb0bf1ff5a6c130e3211b91bca5dca55d0c Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 19 Jul 2022 17:12:55 +0200 Subject: [PATCH 18/36] Update readme file for DLT component --- src/dlt/gateway/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/dlt/gateway/README.md b/src/dlt/gateway/README.md index 505b1c135..71ae3d5eb 100644 --- a/src/dlt/gateway/README.md +++ b/src/dlt/gateway/README.md @@ -73,3 +73,19 @@ Other proto files don't play any significant role and could be safely ignored by ### Client example This code is not necessary to the service, but it could be used to test the service. It contains a sample gRPC client which connects the service and perform all the CRUD operations. + +# Fabric deployment notes + +## General notes +Current Fabric deployment uses Fabric test network with some additional helping scripts on top of it. +This deployment is highly insecure and *shouldn't be used in production* in any case. +To start the network just run the `raft.sh` from `blockchain/scripts` directory. Use `stop.sh` +when you need to stop the network. + +## Server start preparations +To run the server it's necessary to copy certificate file +`fabric-samples/test-network/organizations/peerOrganizations/org1.example.com/ca/ca.org1.example.com-cert.pem` +to the config folder (replacing the existing one). Also it's necessary to copy `scripts/connection-org1.json` +file (again, replacing the old one). After copying, it must be edited. First, all `localhost` entrances +should be replaced with `teraflow.nlehd.de`. Second, `channel` section at the end of the file should be removed. +This should be done after every restart of the Fabric network. \ No newline at end of file -- GitLab From f06ebeb384033a51cdf1c258cb7c74b46813c5aa Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Jul 2022 10:21:10 +0200 Subject: [PATCH 19/36] Use randomized wallet names --- src/dlt/gateway/.gitignore | 2 +- src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dlt/gateway/.gitignore b/src/dlt/gateway/.gitignore index 1de6c650e..9ecdb254c 100644 --- a/src/dlt/gateway/.gitignore +++ b/src/dlt/gateway/.gitignore @@ -87,4 +87,4 @@ gradle-app.setting .gradletasknamecache local.properties -wallet/ \ No newline at end of file +wallet*/ \ No newline at end of file diff --git a/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt index 0ff89ae3e..9b4e1f4dc 100644 --- a/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt +++ b/src/dlt/gateway/src/main/kotlin/grpc/FabricServer.kt @@ -48,8 +48,9 @@ class FabricServer(val port: Int) { private val server: Server init { - val cfg = Config.DltConfig.newBuilder().setWallet("wallet").setConnectionFile("config/connection-org1.json") - .setUser("appUser" + Random.nextUInt()) + val id = Random.nextUInt() + val cfg = Config.DltConfig.newBuilder().setWallet("wallet$id").setConnectionFile("config/connection-org1.json") + .setUser("appUser$id") .setChannel("dlt") .setContract("basic").setCaCertFile("config/ca.org1.example.com-cert.pem").setCaUrl("https://teraflow.nlehd.de:7054") .setCaAdmin("admin").setCaAdminSecret("adminpw").setMsp("Org1MSP").setAffiliation("org1.department1") -- GitLab From b61c2d41c02b1f25dc48b07d34e4b2b38bf0f334 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Jul 2022 15:05:06 +0200 Subject: [PATCH 20/36] DLT server side performs protobuf deserialization --- .../src/main/kotlin/fabric/FabricConnector.kt | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index d175f6011..402741649 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -37,7 +37,9 @@ package fabric -import dlt.DltGateway +import context.ContextOuterClass +import dlt.DltGateway.DltRecord +import dlt.DltGateway.DltRecordEvent import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.runBlocking import org.hyperledger.fabric.gateway.Contract @@ -56,7 +58,7 @@ class FabricConnector(val config: Config.DltConfig) { private val wallet: Wallet private val contract: Contract - private val channels: MutableList> = mutableListOf() + private val channels: MutableList> = mutableListOf() init { // Create a CA client for interacting with the CA. @@ -78,11 +80,29 @@ class FabricConnector(val config: Config.DltConfig) { val consumer = Consumer { event: ContractEvent? -> run { println("new event detected") - val parsedEvent = DltGateway.DltRecordEvent.parseFrom(event?.payload?.get()) - println(parsedEvent.recordId.recordUuid) + val record = DltRecord.parseFrom(event?.payload?.get()) + println(record.recordId.recordUuid) + val eventType: ContextOuterClass.EventTypeEnum = when (event?.name) { + "Add" -> ContextOuterClass.EventTypeEnum.EVENTTYPE_CREATE + "Update" -> ContextOuterClass.EventTypeEnum.EVENTTYPE_UPDATE + "Remove" -> ContextOuterClass.EventTypeEnum.EVENTTYPE_REMOVE + else -> ContextOuterClass.EventTypeEnum.EVENTTYPE_UNDEFINED + } + val pbEvent = DltRecordEvent.newBuilder() + .setEvent( + ContextOuterClass.Event.newBuilder() + .setTimestamp( + ContextOuterClass.Timestamp.newBuilder() + .setTimestamp(System.currentTimeMillis().toDouble()) + ) + .setEventType(eventType) + ) + .setRecordId(record.recordId) + .build() + runBlocking { channels.forEach { - it.trySend(parsedEvent) + it.trySend(pbEvent) } } } @@ -96,49 +116,45 @@ class FabricConnector(val config: Config.DltConfig) { return getContract(config, wallet) } - fun putData(record: DltGateway.DltRecord): String { + fun putData(record: DltRecord): String { println(record.toString()) + println("Put: ${record.toByteArray().decodeToString().length}") return String( contract.submitTransaction( "AddRecord", - record.recordId.domainUuid.uuid, record.recordId.recordUuid.uuid, - record.recordId.type.number.toString(), - record.dataJson + record.toByteArray().decodeToString() ) ) } - fun getData(uuid: String): DltGateway.DltRecord { + fun getData(uuid: String): DltRecord { val result = contract.evaluateTransaction("GetRecord", uuid) - return DltGateway.DltRecord.parseFrom(result) + println("Get: ${result.size}") + return DltRecord.parseFrom(result) } - fun updateData(record: DltGateway.DltRecord): String { + fun updateData(record: DltRecord): String { return String( contract.submitTransaction( "UpdateRecord", - record.recordId.domainUuid.uuid, record.recordId.recordUuid.uuid, - record.recordId.type.number.toString(), - record.dataJson + record.toByteArray().decodeToString() ) ) } - fun deleteData(record: DltGateway.DltRecord): String { + fun deleteData(record: DltRecord): String { return String( contract.submitTransaction( "DeleteRecord", - record.recordId.domainUuid.uuid, record.recordId.recordUuid.uuid, - record.recordId.type.number.toString() ) ) } - fun subscribeForEvents(): Channel { - val produceCh = Channel() + fun subscribeForEvents(): Channel { + val produceCh = Channel() channels.add(produceCh) return produceCh } -- GitLab From c6b89953f9ef5a7d97c8be61a3f66f6460020549 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Jul 2022 15:58:37 +0200 Subject: [PATCH 21/36] Catch exceptions on the server side --- .../src/main/kotlin/fabric/FabricConnector.kt | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index 402741649..53e88a56f 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -119,13 +119,18 @@ class FabricConnector(val config: Config.DltConfig) { fun putData(record: DltRecord): String { println(record.toString()) println("Put: ${record.toByteArray().decodeToString().length}") - return String( + + try { contract.submitTransaction( "AddRecord", record.recordId.recordUuid.uuid, record.toByteArray().decodeToString() ) - ) + } catch (e: Exception) { + println(e.toString()) + return e.toString() + } + return "" } fun getData(uuid: String): DltRecord { @@ -135,22 +140,28 @@ class FabricConnector(val config: Config.DltConfig) { } fun updateData(record: DltRecord): String { - return String( + try { contract.submitTransaction( "UpdateRecord", record.recordId.recordUuid.uuid, record.toByteArray().decodeToString() ) - ) + } catch (e: Exception) { + return e.toString() + } + return "" } fun deleteData(record: DltRecord): String { - return String( + try { contract.submitTransaction( "DeleteRecord", record.recordId.recordUuid.uuid, ) - ) + } catch (e: Exception) { + return e.toString() + } + return "" } fun subscribeForEvents(): Channel { -- GitLab From 63003c25cdd022d4bcbc5a91cfcbd9ff5cdd24e8 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Jul 2022 15:58:55 +0200 Subject: [PATCH 22/36] Update cerificate and connection files --- .../gateway/config/ca.org1.example.com-cert.pem | 16 ++++++++-------- src/dlt/gateway/config/connection-org1.json | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/dlt/gateway/config/ca.org1.example.com-cert.pem b/src/dlt/gateway/config/ca.org1.example.com-cert.pem index 442983f12..16117a52f 100644 --- a/src/dlt/gateway/config/ca.org1.example.com-cert.pem +++ b/src/dlt/gateway/config/ca.org1.example.com-cert.pem @@ -1,14 +1,14 @@ -----BEGIN CERTIFICATE----- -MIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw +MIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw cDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH EwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh -Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw +Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw WjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV BAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT -Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk -+U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO -PXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG -AQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD -AgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp -rPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k= +Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb +gNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK +xkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG +AQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD +AgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu +nj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q= -----END CERTIFICATE----- diff --git a/src/dlt/gateway/config/connection-org1.json b/src/dlt/gateway/config/connection-org1.json index 068404e7b..71c68c164 100644 --- a/src/dlt/gateway/config/connection-org1.json +++ b/src/dlt/gateway/config/connection-org1.json @@ -26,7 +26,7 @@ "peer0.org1.example.com": { "url": "grpcs://teraflow.nlehd.de:7051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk\n+U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO\nPXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD\nAgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp\nrPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k=\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb\ngNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK\nxkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD\nAgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu\nnj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q=\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org1.example.com", @@ -36,7 +36,7 @@ "peer0.org2.example.com": { "url": "grpcs://teraflow.nlehd.de:9051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUfNAV+OorBxKgF5kz/zfKH9s8vsowCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MTkxMzAwMDBaFw0zNzA3MTUxMzAwMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQo2aut3d3aKCLs\n6jf+GBOQ+LoTJG1WtTJXCYfwz3UeBqnDPs6XdADQ+tPAB3xLdAqbDLAdniTdJ5uI\n7iD5ujeZo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUb56rcdREBjw0Y2DFn1xrwOnIroUwCgYIKoZIzj0EAwIDRwAwRAIg\nONguVx49qjxsiw9Bdpqmqopd7nV6NczIBAtt/9U2mfcCIEePn5jGZaoJVPpkgN1D\nXgy1XywDqFL05nY/gDnDTJFo\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUbVkhdL6a3cBqIyPyUxvEPr4jc4cwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MjAxMTE2MDBaFw0zNzA3MTYxMTE2MDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARV7ezedZN+/fF7\nyxxCw3h1W5s5po39zG7iIOZetqFV54HxPUgG4qvIoBSrRrUU6iZaPx/Vtzw/6JHb\nuqH1xprPo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUfOYOFd9I3eFKBfRX3k7NWmgz+howCgYIKoZIzj0EAwIDRwAwRAIg\nOUVRdziuF4Tkl9d7dECvbqAL0QkEvC78FKDwpqCRB9gCIEcOwN8r3mwt8NrPny0X\nteQyRXRjPG3QdU93TgpH3R/R\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org2.example.com", @@ -50,7 +50,7 @@ "caName": "ca-org1", "tlsCACerts": { "pem": [ - "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUe3Wp/+sgV9FnubOi/El3IqRgS7IwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzE5MTMwMDAwWhcNMzcwNzE1MTMwMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPJk\n+U2t5Wj/Qtpc7chEDIBNzAXMixqGNA2F8PFoMSIh+x3UUP3Y87OwWFeFSWjdQ2EO\nPXdJw8FRCGiafsqbDE2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBSueKRHoZ8TTpExFhdRU5n0Jf0H2DAKBggqhkjOPQQD\nAgNHADBEAiATpStU+jNE/iMw7nKLkWwOXS4qqvrkm7G927edB8cpQQIgBkS7DVmp\nrPDUeHCGmNdJLVwPM6ernEbtua5QAJ11A0k=\n-----END CERTIFICATE-----\n" + "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb\ngNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK\nxkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD\nAgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu\nnj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q=\n-----END CERTIFICATE-----\n" ] }, "httpOptions": { @@ -62,7 +62,7 @@ "orderer0.example.com": { "url": "grpcs://teraflow.nlehd.de:7050", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUAnklc7GR28OpDSqO+beAokDGNjUwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcxOTEzMDAwMFoXDTM3MDcxNTEzMDAwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEfVGvu1yLzzOuv0xqdLozKOrGIEHjmv0UZ1/IEznG\nbxygeZDsmsagQEJFRPSG9SnrE9XL4qnwRYHAEOyi5hVHGqNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFIjQ/caW9qF+Db2J\nxOtvNvlg2nnxMAoGCCqGSM49BAMCA0cAMEQCIH3rEsvHrUrrEgnVvJmo7xJg5MaR\nvhKqcCC0oNglr8CNAiB6iXUVuXirVF7z3QaUp6DSxuNEHoH6P916pjhEs2U5oA==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUTXs/ZHEfPp7uSOdAsc6JZMn7cX0wCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcyMDExMTYwMFoXDTM3MDcxNjExMTYwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEj+bnBfSoOcB5ZA1gN41GRHrt/iuIEFGeO+jcf3Wq\nrruoGMnUJ0vr0vuoWOmiWGMzvC3BCMRlidUBT70A1jlgR6NFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFKfdYK58sI1GyCZ9\n+886scCKUx6eMAoGCCqGSM49BAMCA0cAMEQCIEpr+FrW0eLLSfOTsuG7mpn0L2Fo\nQsjbtoEOT66dmiW4AiA6pkalgEWCWs40d99IMdNSUObYP+7lf94QOhaWRQrHQQ==\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "orderer0.example.com", -- GitLab From a90dde4ec195ed1f61e01708fd302e6e99e428ad Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Wed, 20 Jul 2022 14:41:15 +0000 Subject: [PATCH 23/36] DLT component: - corrected method used to get record from DLT in test script --- src/dlt/connector/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dlt/connector/main_test.py b/src/dlt/connector/main_test.py index 632497220..4ad90eb35 100644 --- a/src/dlt/connector/main_test.py +++ b/src/dlt/connector/main_test.py @@ -2,7 +2,7 @@ # PYTHONPATH=/home/cttc/teraflow/src python -m dlt.connector.main_test import logging, sys, time -from common.proto.dlt_gateway_pb2 import DLTRECORDOPERATION_ADD, DLTRECORDTYPE_DEVICE, DltRecord +from common.proto.dlt_gateway_pb2 import DLTRECORDOPERATION_ADD, DLTRECORDOPERATION_UPDATE, DLTRECORDTYPE_DEVICE, DltRecord from common.tools.object_factory.Device import json_device from common.tools.grpc.Tools import grpc_message_to_json_string from src.common.proto.context_pb2 import DEVICEOPERATIONALSTATUS_ENABLED, Device @@ -33,7 +33,7 @@ def main(): dlt2r_req = r2dlt_req.record_id LOGGER.info('dlt2r_req = {:s}'.format(grpc_message_to_json_string(dlt2r_req))) - dlt2r_rep = dltgateway_client.RecordToDlt(dlt2r_req) + dlt2r_rep = dltgateway_client.GetFromDlt(dlt2r_req) LOGGER.info('dlt2r_rep = {:s}'.format(grpc_message_to_json_string(dlt2r_rep))) dltgateway_collector.stop() -- GitLab From 1626969632f5ee689e54193bb2a6b45deda65c5f Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Jul 2022 16:44:15 +0200 Subject: [PATCH 24/36] Return empty DltRecord when no data was recorded in blockchain --- .../gateway/src/main/kotlin/fabric/FabricConnector.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index 53e88a56f..d6feaa0ab 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -134,9 +134,14 @@ class FabricConnector(val config: Config.DltConfig) { } fun getData(uuid: String): DltRecord { - val result = contract.evaluateTransaction("GetRecord", uuid) - println("Get: ${result.size}") - return DltRecord.parseFrom(result) + try { + val result = contract.evaluateTransaction("GetRecord", uuid) + println("Get: ${result.size}") + return DltRecord.parseFrom(result) + } catch (e: Exception) { + println(e.toString()) + return DltRecord.getDefaultInstance() + } } fun updateData(record: DltRecord): String { -- GitLab From 3b95df2969716cc86d8bfbb7c62a33bcdb910b8d Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Thu, 21 Jul 2022 10:42:46 +0200 Subject: [PATCH 25/36] Encode protobuf payloads as Base64 strings before to put them into Fabric --- src/dlt/gateway/src/main/kotlin/Main.kt | 7 ++++++- .../src/main/kotlin/fabric/FabricConnector.kt | 17 +++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/dlt/gateway/src/main/kotlin/Main.kt b/src/dlt/gateway/src/main/kotlin/Main.kt index 576aeee56..c57c9e980 100644 --- a/src/dlt/gateway/src/main/kotlin/Main.kt +++ b/src/dlt/gateway/src/main/kotlin/Main.kt @@ -118,7 +118,12 @@ fun main() = runBlocking { val data = DltGateway.DltRecord.newBuilder() .setRecordId(id) .setOperation(DltGateway.DltRecordOperationEnum.DLTRECORDOPERATION_ADD) - .setDataJson("{}") + .setDataJson("\"{\"device_config\": {\"config_rules\": []}, \"device_drivers\": []," + + "\"device_endpoints\": [], \"device_id\": {\"device_uuid\": {\"uuid\": \"dev-12345\"}}," + + "\"device_operational_status\": \"DEVICEOPERATIONALSTATUS_ENABLED\"," + + "\"device_type\": \"packet-router\"}\", \"operation\": \"DLTRECORDOPERATION_ADD\"," + + "\"record_id\": {\"domain_uuid\": {\"uuid\": \"tfs-a\"}, \"record_uuid\": {\"uuid\": \"dev-12345\"}," + + "\"type\": \"DLTRECORDTYPE_DEVICE\"}") .build() println("sending new record") diff --git a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt index d6feaa0ab..af6592be9 100644 --- a/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt +++ b/src/dlt/gateway/src/main/kotlin/fabric/FabricConnector.kt @@ -60,6 +60,9 @@ class FabricConnector(val config: Config.DltConfig) { private val channels: MutableList> = mutableListOf() + private val encoder: Base64.Encoder = Base64.getEncoder() + private val decoder: Base64.Decoder = Base64.getDecoder() + init { // Create a CA client for interacting with the CA. val props = Properties() @@ -80,7 +83,7 @@ class FabricConnector(val config: Config.DltConfig) { val consumer = Consumer { event: ContractEvent? -> run { println("new event detected") - val record = DltRecord.parseFrom(event?.payload?.get()) + val record = DltRecord.parseFrom(decoder.decode(event?.payload?.get())) println(record.recordId.recordUuid) val eventType: ContextOuterClass.EventTypeEnum = when (event?.name) { "Add" -> ContextOuterClass.EventTypeEnum.EVENTTYPE_CREATE @@ -118,13 +121,12 @@ class FabricConnector(val config: Config.DltConfig) { fun putData(record: DltRecord): String { println(record.toString()) - println("Put: ${record.toByteArray().decodeToString().length}") try { contract.submitTransaction( "AddRecord", record.recordId.recordUuid.uuid, - record.toByteArray().decodeToString() + encoder.encodeToString(record.toByteArray()) ) } catch (e: Exception) { println(e.toString()) @@ -134,13 +136,12 @@ class FabricConnector(val config: Config.DltConfig) { } fun getData(uuid: String): DltRecord { - try { + return try { val result = contract.evaluateTransaction("GetRecord", uuid) - println("Get: ${result.size}") - return DltRecord.parseFrom(result) + DltRecord.parseFrom(decoder.decode(result)) } catch (e: Exception) { println(e.toString()) - return DltRecord.getDefaultInstance() + DltRecord.getDefaultInstance() } } @@ -149,7 +150,7 @@ class FabricConnector(val config: Config.DltConfig) { contract.submitTransaction( "UpdateRecord", record.recordId.recordUuid.uuid, - record.toByteArray().decodeToString() + encoder.encodeToString(record.toByteArray()) ) } catch (e: Exception) { return e.toString() -- GitLab From d3fc5065042f736cb001357431a594258591051d Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Fri, 19 Aug 2022 18:13:28 +0200 Subject: [PATCH 26/36] Readme for DLT component: add information about Fabric configuration --- src/dlt/gateway/README.md | 49 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/dlt/gateway/README.md b/src/dlt/gateway/README.md index 71ae3d5eb..2cf6cfeb1 100644 --- a/src/dlt/gateway/README.md +++ b/src/dlt/gateway/README.md @@ -78,14 +78,57 @@ a sample gRPC client which connects the service and perform all the CRUD operati ## General notes Current Fabric deployment uses Fabric test network with some additional helping scripts on top of it. -This deployment is highly insecure and *shouldn't be used in production* in any case. To start the network just run the `raft.sh` from `blockchain/scripts` directory. Use `stop.sh` when you need to stop the network. ## Server start preparations To run the server it's necessary to copy certificate file `fabric-samples/test-network/organizations/peerOrganizations/org1.example.com/ca/ca.org1.example.com-cert.pem` -to the config folder (replacing the existing one). Also it's necessary to copy `scripts/connection-org1.json` +to the config folder (replacing the existing one). Also, it's necessary to copy `scripts/connection-org1.json` file (again, replacing the old one). After copying, it must be edited. First, all `localhost` entrances should be replaced with `teraflow.nlehd.de`. Second, `channel` section at the end of the file should be removed. -This should be done after every restart of the Fabric network. \ No newline at end of file +This should be done after every restart of the Fabric network. + +## Fabric configuration +Even though a test network is easy to deploy and use it's better to perform a custom configuration +for a production deployment. In practice every participating organization will likely prefer to have +its own Peer/Orderer/CA instances to prevent possible dependency on any other participants. This leads +not only to a better privacy/availability/security in general but also to the more complicated +deployment process as a side effect. Here we provide a very brief description of the most important points. + +### Organizations +Organization represents a network participant, which can be an individual, a large corporation or any other +entity. Each organization has its own CAs, orderers and peers. The recommendation here is to create an +organization entity for every independent participant and then decide how many CAs/peers/orderers does +every organization need and which channels should it has access to based on the exact project's goals. + +### Channels +Each channel represents an independent ledger with its own genesis block. Each transaction is executed +on a specific channel, and it's possible to define which organization has access to a given channel. +As a result channels are a pretty powerful privacy mechanism which allows to limit access to the private +data between organization. + +### Certificate authorities, peers and orderers +Certificate authorities (CA) are used to generate crypto materials for each organization. Two types of CA +exist: one is used to generate the certificates of the admin, the MSP and certificates of non-admin users. +Another type of CA is used to generate TLS certificates. As a result it's preferable to have at least two +CAs for every organization. + +Peers are entities which host ledgers and smart contracts. They communicate with applications and orderers, +receiving chaincode invocations (proposals), invoking chaincode, updating ledger when necessary and +returning result of execution. Peers can handle one or many ledgers, depending on the configuration. It's +very use case specific how many peers are necessary to the exact deployment. + +Orderers are used to execute a consensus in a distributing network making sure that every channel participant +has the same blocks with the same data. The default consensus algorithm is Raft which provides only a crash +fault tolerance. + +### Conclusion +As you can see, configuration procedure for Fabric is pretty tricky and includes quite a lot of entities. +In real world it will very likely involve participants from multiple organizations each of them performing +its own part of configuration. + +As a further reading it's recommended to start with the +[official deployment guide](https://hyperledger-fabric.readthedocs.io/en/release-2.2/deployment_guide_overview.html). +It contains a high level overview of a deployment process as well as links to the detailed descriptions to +CA/Peer/Orderer configuration descriptions. \ No newline at end of file -- GitLab From d179bf51fb8b75c063037370f7f58eb386f28faf Mon Sep 17 00:00:00 2001 From: Sebastien Andreina Date: Tue, 27 Sep 2022 11:03:51 +0200 Subject: [PATCH 27/36] updated configuration file after reboot --- .../gateway/config/ca.org1.example.com-cert.pem | 16 ++++++++-------- src/dlt/gateway/config/connection-org1.json | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/dlt/gateway/config/ca.org1.example.com-cert.pem b/src/dlt/gateway/config/ca.org1.example.com-cert.pem index 16117a52f..d7fdf63cc 100644 --- a/src/dlt/gateway/config/ca.org1.example.com-cert.pem +++ b/src/dlt/gateway/config/ca.org1.example.com-cert.pem @@ -1,14 +1,14 @@ -----BEGIN CERTIFICATE----- -MIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw +MIICJzCCAc2gAwIBAgIUb5gDMfVeVdQjFkK3uC8LtlogN+gwCgYIKoZIzj0EAwIw cDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH EwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh -Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw +Lm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwOTI3MDgzMDAwWhcNMzcwOTIzMDgzMDAw WjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV BAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT -Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb -gNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK -xkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG -AQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD -AgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu -nj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q= +Y2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDC3 +spCTT3pjfFXxkX/SFuBgWRiceR8rSoCNQOnIPeNGZK8xl2Zr7VuY06gqy9c+ecSU +PUWaXiCQxiLgZuS6TOWjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG +AQH/AgEBMB0GA1UdDgQWBBRFWSc7GZqcJJyJjXSEspzgAYInGzAKBggqhkjOPQQD +AgNIADBFAiEAodqc+adkiMuU6iv1IF8uJ/nMQbvMGoP3pb2827QzDosCICOw6W+y +uH03H3RO6KhOcS1ZzPjspyjrcC+dwzYX4DpW -----END CERTIFICATE----- diff --git a/src/dlt/gateway/config/connection-org1.json b/src/dlt/gateway/config/connection-org1.json index 71c68c164..6f6f3f08d 100644 --- a/src/dlt/gateway/config/connection-org1.json +++ b/src/dlt/gateway/config/connection-org1.json @@ -26,7 +26,7 @@ "peer0.org1.example.com": { "url": "grpcs://teraflow.nlehd.de:7051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb\ngNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK\nxkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD\nAgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu\nnj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q=\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUb5gDMfVeVdQjFkK3uC8LtlogN+gwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwOTI3MDgzMDAwWhcNMzcwOTIzMDgzMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDC3\nspCTT3pjfFXxkX/SFuBgWRiceR8rSoCNQOnIPeNGZK8xl2Zr7VuY06gqy9c+ecSU\nPUWaXiCQxiLgZuS6TOWjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRFWSc7GZqcJJyJjXSEspzgAYInGzAKBggqhkjOPQQD\nAgNIADBFAiEAodqc+adkiMuU6iv1IF8uJ/nMQbvMGoP3pb2827QzDosCICOw6W+y\nuH03H3RO6KhOcS1ZzPjspyjrcC+dwzYX4DpW\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org1.example.com", @@ -36,7 +36,7 @@ "peer0.org2.example.com": { "url": "grpcs://teraflow.nlehd.de:9051", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUbVkhdL6a3cBqIyPyUxvEPr4jc4cwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA3MjAxMTE2MDBaFw0zNzA3MTYxMTE2MDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARV7ezedZN+/fF7\nyxxCw3h1W5s5po39zG7iIOZetqFV54HxPUgG4qvIoBSrRrUU6iZaPx/Vtzw/6JHb\nuqH1xprPo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUfOYOFd9I3eFKBfRX3k7NWmgz+howCgYIKoZIzj0EAwIDRwAwRAIg\nOUVRdziuF4Tkl9d7dECvbqAL0QkEvC78FKDwpqCRB9gCIEcOwN8r3mwt8NrPny0X\nteQyRXRjPG3QdU93TgpH3R/R\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICHjCCAcWgAwIBAgIUL48scgv9ItATkBjSNhzYDjLUDsAwCgYIKoZIzj0EAwIw\nbDELMAkGA1UEBhMCVUsxEjAQBgNVBAgTCUhhbXBzaGlyZTEQMA4GA1UEBxMHSHVy\nc2xleTEZMBcGA1UEChMQb3JnMi5leGFtcGxlLmNvbTEcMBoGA1UEAxMTY2Eub3Jn\nMi5leGFtcGxlLmNvbTAeFw0yMjA5MjcwODMwMDBaFw0zNzA5MjMwODMwMDBaMGwx\nCzAJBgNVBAYTAlVLMRIwEAYDVQQIEwlIYW1wc2hpcmUxEDAOBgNVBAcTB0h1cnNs\nZXkxGTAXBgNVBAoTEG9yZzIuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2NhLm9yZzIu\nZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQ5qz8FfrEQ5S08\nr/avPyTrF2grXj5L4DnbvF4YEZ5Usnbm8Svovu7PO8uiVcwT5vrt6ssOdpBFZYu3\nNndpojnYo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAd\nBgNVHQ4EFgQUYcp7axYV9AaIptYQqhiCL0VDmXQwCgYIKoZIzj0EAwIDRwAwRAIg\nWT1V8/6flUPNcBkmbtEEKf83k7+6sR9k1a2wtVeJFnQCIE0ZSIL3k0dKQydQBpiz\nPcZZUULvQivcMlIsw5+mjIGc\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "peer0.org2.example.com", @@ -50,7 +50,7 @@ "caName": "ca-org1", "tlsCACerts": { "pem": [ - "-----BEGIN CERTIFICATE-----\nMIICJjCCAc2gAwIBAgIUcaqBwLFi7Vi26qOKsAhI0jPtxuQwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwNzIwMTExNjAwWhcNMzcwNzE2MTExNjAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABEdb\ngNA5a+J/ieX2k1HyjxCYYkljbYQW3dAm0GQGEZ0r7OEzkdU6TLPeMYq25q+5rLOK\nxkEng/D1BTy29jsO8s2jRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBS2Yd1utSIM/WzVO+qTF0dLLQjN/zAKBggqhkjOPQQD\nAgNHADBEAiAxpw295JN9x8WFhEYaCEXdRj9TCLgJZTo3hpmMloqcfgIgYxRL4geu\nnj2/9fJPU7GEsDfF6sySVceGUwtQMWlbR5Q=\n-----END CERTIFICATE-----\n" + "-----BEGIN CERTIFICATE-----\nMIICJzCCAc2gAwIBAgIUb5gDMfVeVdQjFkK3uC8LtlogN+gwCgYIKoZIzj0EAwIw\ncDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYDVQQH\nEwZEdXJoYW0xGTAXBgNVBAoTEG9yZzEuZXhhbXBsZS5jb20xHDAaBgNVBAMTE2Nh\nLm9yZzEuZXhhbXBsZS5jb20wHhcNMjIwOTI3MDgzMDAwWhcNMzcwOTIzMDgzMDAw\nWjBwMQswCQYDVQQGEwJVUzEXMBUGA1UECBMOTm9ydGggQ2Fyb2xpbmExDzANBgNV\nBAcTBkR1cmhhbTEZMBcGA1UEChMQb3JnMS5leGFtcGxlLmNvbTEcMBoGA1UEAxMT\nY2Eub3JnMS5leGFtcGxlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDC3\nspCTT3pjfFXxkX/SFuBgWRiceR8rSoCNQOnIPeNGZK8xl2Zr7VuY06gqy9c+ecSU\nPUWaXiCQxiLgZuS6TOWjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG\nAQH/AgEBMB0GA1UdDgQWBBRFWSc7GZqcJJyJjXSEspzgAYInGzAKBggqhkjOPQQD\nAgNIADBFAiEAodqc+adkiMuU6iv1IF8uJ/nMQbvMGoP3pb2827QzDosCICOw6W+y\nuH03H3RO6KhOcS1ZzPjspyjrcC+dwzYX4DpW\n-----END CERTIFICATE-----\n" ] }, "httpOptions": { @@ -62,7 +62,7 @@ "orderer0.example.com": { "url": "grpcs://teraflow.nlehd.de:7050", "tlsCACerts": { - "pem": "-----BEGIN CERTIFICATE-----\nMIICCjCCAbGgAwIBAgIUTXs/ZHEfPp7uSOdAsc6JZMn7cX0wCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDcyMDExMTYwMFoXDTM3MDcxNjExMTYwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAEj+bnBfSoOcB5ZA1gN41GRHrt/iuIEFGeO+jcf3Wq\nrruoGMnUJ0vr0vuoWOmiWGMzvC3BCMRlidUBT70A1jlgR6NFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFKfdYK58sI1GyCZ9\n+886scCKUx6eMAoGCCqGSM49BAMCA0cAMEQCIEpr+FrW0eLLSfOTsuG7mpn0L2Fo\nQsjbtoEOT66dmiW4AiA6pkalgEWCWs40d99IMdNSUObYP+7lf94QOhaWRQrHQQ==\n-----END CERTIFICATE-----\n" + "pem": "-----BEGIN CERTIFICATE-----\nMIICCzCCAbGgAwIBAgIUdZQo3q4OqyxIkidmAV4QkewCylIwCgYIKoZIzj0EAwIw\nYjELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcg\nWW9yazEUMBIGA1UEChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUu\nY29tMB4XDTIyMDkyNzA4MzAwMFoXDTM3MDkyMzA4MzAwMFowYjELMAkGA1UEBhMC\nVVMxETAPBgNVBAgTCE5ldyBZb3JrMREwDwYDVQQHEwhOZXcgWW9yazEUMBIGA1UE\nChMLZXhhbXBsZS5jb20xFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMFkwEwYHKoZI\nzj0CAQYIKoZIzj0DAQcDQgAERR0UzsHSFoyON+9Noxmk1IhnTvSdLWGgEpEwrqVr\n5DwitkeJwRWq134JBTmXuZzsUG87oN6Hr94XAEe4j9Zq8qNFMEMwDgYDVR0PAQH/\nBAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFN8XsELp/X0akrlJ\nY3/BWo2jZS3cMAoGCCqGSM49BAMCA0gAMEUCIQCZYYXW/0h3Kq4BmROpOHfrondg\nopf5LndeujYlH3i8tQIgCtpTQiDXZd+IAUduRmn7a46CwJSbjYbXFVX5vumIbE4=\n-----END CERTIFICATE-----\n" }, "grpcOptions": { "ssl-target-name-override": "orderer0.example.com", -- GitLab From 310f86cf6cc569525c7b048bfe54c2101adf657b Mon Sep 17 00:00:00 2001 From: Lluis Gifre Date: Tue, 27 Sep 2022 10:48:53 +0000 Subject: [PATCH 28/36] Added DLT manifest file --- manifests/dltservice.yaml | 86 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 manifests/dltservice.yaml diff --git a/manifests/dltservice.yaml b/manifests/dltservice.yaml new file mode 100644 index 000000000..5ef6eae7d --- /dev/null +++ b/manifests/dltservice.yaml @@ -0,0 +1,86 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dltservice +spec: + selector: + matchLabels: + app: dltservice + template: + metadata: + labels: + app: dltservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: connector + image: registry.gitlab.com/teraflow-h2020/controller/dlt-connector:latest + imagePullPolicy: Always + ports: + - containerPort: 8080 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi + - name: gateway + image: registry.gitlab.com/teraflow-h2020/controller/dlt-gateway:latest + imagePullPolicy: Always + #readinessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + #livenessProbe: + # httpGet: + # path: /health + # port: 8081 + # initialDelaySeconds: 5 + # timeoutSeconds: 5 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: dltservice +spec: + type: ClusterIP + selector: + app: dltservice + ports: + - name: grpc + protocol: TCP + port: 8080 + targetPort: 8080 -- GitLab From 11ee5313d51bb66cff1d12b932be0e9f29ac2b07 Mon Sep 17 00:00:00 2001 From: Sebastien Andreina Date: Wed, 28 Sep 2022 09:10:50 +0200 Subject: [PATCH 29/36] added dockerfile for kub deployment --- src/dlt/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/dlt/Dockerfile diff --git a/src/dlt/Dockerfile b/src/dlt/Dockerfile new file mode 100644 index 000000000..aee2c5e89 --- /dev/null +++ b/src/dlt/Dockerfile @@ -0,0 +1,12 @@ +# To add to host when starting container +# 195.37.154.24 peer0.org1.example.com +# 195.37.154.24 peer0.org2.example.com +# 195.37.154.24 orderer0.example.com +FROM zenika/kotlin + +# Copy controller project +COPY . /controller + +# Build gateway +WORKDIR /controller/src/dlt/gateway +RUN ./gradlew build -- GitLab From 027fc68760d6916f053133cf39d315d4b9401717 Mon Sep 17 00:00:00 2001 From: hallo Date: Wed, 28 Sep 2022 16:20:15 +0200 Subject: [PATCH 30/36] updated dockerfile to add automated service start --- src/dlt/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/dlt/Dockerfile b/src/dlt/Dockerfile index aee2c5e89..b0485d99f 100644 --- a/src/dlt/Dockerfile +++ b/src/dlt/Dockerfile @@ -10,3 +10,16 @@ COPY . /controller # Build gateway WORKDIR /controller/src/dlt/gateway RUN ./gradlew build + +EXPOSE 50051 + +RUN echo "#!/bin/sh" >> /runscript.sh +RUN echo "echo 195.37.154.24 peer0.org1.example.com >> /etc/hosts;\ + echo 195.37.154.24 peer0.org2.example.com >> /etc/hosts;\ + echo 195.37.154.24 orderer0.example.com >> /etc/hosts;" >> /runscript.sh + +RUN echo "cd /controller/src/dlt/gateway; ./gradlew runServer" >> /runscript.sh +RUN chmod +x /runscript.sh + + +ENTRYPOINT ["sh", "/runscript.sh"] -- GitLab From 7908c61bcf8efa70f3be130bcff768038b61ea51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Andreina?= Date: Wed, 28 Sep 2022 15:36:46 +0000 Subject: [PATCH 31/36] Cleanup file --- src/dlt/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dlt/Dockerfile b/src/dlt/Dockerfile index b0485d99f..bbf21eb32 100644 --- a/src/dlt/Dockerfile +++ b/src/dlt/Dockerfile @@ -1,7 +1,3 @@ -# To add to host when starting container -# 195.37.154.24 peer0.org1.example.com -# 195.37.154.24 peer0.org2.example.com -# 195.37.154.24 orderer0.example.com FROM zenika/kotlin # Copy controller project -- GitLab From 0a1b61788093422dc2ecda8a1d13dec2ae813924 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 27 Oct 2022 16:06:26 +0000 Subject: [PATCH 32/36] DLT: MockBlockchain: - Implemented mock-blockchain module to facilitate testing and debugging of DLT connector & workflows - Added manifests, Dockerfile, etc. DLT Connector: - corrected requirements.in - updated Dockerfile - WIP: implementation of test_unitary.py DLT Gateway: - Improved/extended Dockerfile - Moved Dockerfile to correct folder - Corrected import path for proto files --- .gitignore | 1 + deploy_mock_blockchain.sh | 121 ++++++++++++++++++ manifests/mock_blockchain.yaml | 64 +++++++++ src/common/Constants.py | 6 + .../tests/MockServicerImpl_DltGateway.py | 6 +- src/dlt/Dockerfile | 21 --- src/dlt/connector/Dockerfile | 65 +++++++--- src/dlt/connector/requirements.in | 6 - src/dlt/connector/tests/test_unitary.py | 21 ++- src/dlt/gateway/Dockerfile | 41 ++++++ src/dlt/gateway/build.gradle.kts | 2 +- src/dlt/mock_blockchain/Dockerfile | 68 ++++++++++ src/dlt/mock_blockchain/__init__.py | 13 ++ src/dlt/mock_blockchain/requirements.in | 0 src/dlt/mock_blockchain/service/__init__.py | 13 ++ src/dlt/mock_blockchain/service/__main__.py | 61 +++++++++ 16 files changed, 456 insertions(+), 53 deletions(-) create mode 100755 deploy_mock_blockchain.sh create mode 100644 manifests/mock_blockchain.yaml delete mode 100644 src/dlt/Dockerfile create mode 100644 src/dlt/gateway/Dockerfile create mode 100644 src/dlt/mock_blockchain/Dockerfile create mode 100644 src/dlt/mock_blockchain/__init__.py create mode 100644 src/dlt/mock_blockchain/requirements.in create mode 100644 src/dlt/mock_blockchain/service/__init__.py create mode 100644 src/dlt/mock_blockchain/service/__main__.py diff --git a/.gitignore b/.gitignore index 5dc4372a5..7e3b0cd6a 100644 --- a/.gitignore +++ b/.gitignore @@ -162,6 +162,7 @@ cython_debug/ # TeraFlowSDN-generated files tfs_runtime_env_vars.sh +tfs_bchain_runtime_env_vars.sh delete_local_deployment.sh local_docker_deployment.sh local_k8s_deployment.sh diff --git a/deploy_mock_blockchain.sh b/deploy_mock_blockchain.sh new file mode 100755 index 000000000..066820fc0 --- /dev/null +++ b/deploy_mock_blockchain.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +######################################################################################################################## +# Read deployment settings +######################################################################################################################## + +# Set the URL of your local Docker registry where the images will be uploaded to. +REGISTRY_IMAGE="http://localhost:32000/tfs/" + +# Set the tag you want to use for your images. +IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy to. +K8S_NAMESPACE="tfs-bchain" + +COMPONENT="mock_blockchain" + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +# Constants +GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller" +TMP_FOLDER="./tmp" + +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" +mkdir -p $TMP_MANIFESTS_FOLDER +TMP_LOGS_FOLDER="$TMP_FOLDER/logs" +mkdir -p $TMP_LOGS_FOLDER + +echo "Deleting and Creating a new namespace..." +kubectl delete namespace $K8S_NAMESPACE +kubectl create namespace $K8S_NAMESPACE +printf "\n" + +echo "Deploying components and collecting environment variables..." +ENV_VARS_SCRIPT=tfs_bchain_runtime_env_vars.sh +echo "# Environment variables for TeraFlow Mock-Blockchain deployment" > $ENV_VARS_SCRIPT +PYTHONPATH=$(pwd)/src + +echo "Processing '$COMPONENT' component..." +IMAGE_NAME="$COMPONENT:$IMAGE_TAG" +IMAGE_URL=$(echo "$REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') + +echo " Building Docker image..." +BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" +docker build -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG" + +if [ -n "$REGISTRY_IMAGE" ]; then + echo " Pushing Docker image to '$REGISTRY_IMAGE'..." + + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" + docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" + + PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" + docker push "$IMAGE_URL" > "$PUSH_LOG" +fi + +echo " Adapting '$COMPONENT' manifest file..." +MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}.yaml" +cp ./manifests/"${COMPONENT}".yaml "$MANIFEST" + +if [ -n "$REGISTRY_IMAGE" ]; then + # Registry is set + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" + sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" +else + # Registry is not set + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" + sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" +fi + +echo " Deploying '$COMPONENT' component to Kubernetes..." +DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log" +kubectl --namespace $K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG" +COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") +kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" +kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" + +echo " Collecting env-vars for '$COMPONENT' component..." +SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME} --namespace $K8S_NAMESPACE -o json) + +# Env vars for service's host address +SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP') +ENVVAR_HOST=$(echo "${COMPONENT}_SERVICE_HOST" | tr '[:lower:]' '[:upper:]') +echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT + +# Env vars for service's 'grpc' port +SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port') +ENVVAR_PORT_GRPC=$(echo "${COMPONENT}_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]') +echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT + +printf "\n" + +echo "Waiting for '$COMPONENT' component..." +kubectl wait --namespace $K8S_NAMESPACE \ + --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME} +printf "\n" + +echo "Deployment Resources:" +kubectl --namespace $K8S_NAMESPACE get all +printf "\n" + +echo "Done!" diff --git a/manifests/mock_blockchain.yaml b/manifests/mock_blockchain.yaml new file mode 100644 index 000000000..b383d7db4 --- /dev/null +++ b/manifests/mock_blockchain.yaml @@ -0,0 +1,64 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-blockchain +spec: + selector: + matchLabels: + app: mock-blockchain + template: + metadata: + labels: + app: mock-blockchain + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: registry.gitlab.com/teraflow-h2020/controller/mock_blockchain:latest + imagePullPolicy: Always + ports: + - containerPort: 50051 + env: + - name: LOG_LEVEL + value: "DEBUG" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: mock-blockchain +spec: + type: ClusterIP + selector: + app: mock-blockchain + ports: + - name: grpc + protocol: TCP + port: 50051 + targetPort: 50051 diff --git a/src/common/Constants.py b/src/common/Constants.py index f18d43840..a536ef600 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -49,6 +49,9 @@ class ServiceNameEnum(Enum): PATHCOMP = 'pathcomp' WEBUI = 'webui' + # Used for test and debugging only + DLT_GATEWAY = 'dlt-gateway' + # Default gRPC service ports DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.CONTEXT .value : 1010, @@ -63,6 +66,9 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.CYBERSECURITY.value : 10000, ServiceNameEnum.INTERDOMAIN .value : 10010, ServiceNameEnum.PATHCOMP .value : 10020, + + # Used for test and debugging only + ServiceNameEnum.DLT_GATEWAY .value : 50051, } # Default HTTP/REST-API service ports diff --git a/src/common/tests/MockServicerImpl_DltGateway.py b/src/common/tests/MockServicerImpl_DltGateway.py index 16eae7a34..2d7501682 100644 --- a/src/common/tests/MockServicerImpl_DltGateway.py +++ b/src/common/tests/MockServicerImpl_DltGateway.py @@ -13,7 +13,7 @@ # limitations under the License. import grpc, itertools, json, logging, time -from typing import Dict, Iterator, Optional, Tuple +from typing import Any, Dict, Iterator, Optional, Tuple from common.tests.MockMessageBroker import Message, MockMessageBroker from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import EVENTTYPE_CREATE, EVENTTYPE_REMOVE, EVENTTYPE_UPDATE, Empty, TeraFlowController @@ -27,8 +27,8 @@ from common.proto.dlt_gateway_pb2_grpc import DltGatewayServiceServicer LOGGER = logging.getLogger(__name__) -DltRecordKey = Tuple[str, DltRecordOperationEnum, str] # domain_uuid, operation, record_uuid -DltRecordDict = Dict[DltRecordKey, DltRecord] # dlt_record_key => dlt_record +DltRecordKey = Tuple[str, Any, str] # domain_uuid, DltRecordOperationEnum, record_uuid +DltRecordDict = Dict[DltRecordKey, DltRecord] # dlt_record_key => dlt_record class AlreadyExistsException(Exception): pass diff --git a/src/dlt/Dockerfile b/src/dlt/Dockerfile deleted file mode 100644 index bbf21eb32..000000000 --- a/src/dlt/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM zenika/kotlin - -# Copy controller project -COPY . /controller - -# Build gateway -WORKDIR /controller/src/dlt/gateway -RUN ./gradlew build - -EXPOSE 50051 - -RUN echo "#!/bin/sh" >> /runscript.sh -RUN echo "echo 195.37.154.24 peer0.org1.example.com >> /etc/hosts;\ - echo 195.37.154.24 peer0.org2.example.com >> /etc/hosts;\ - echo 195.37.154.24 orderer0.example.com >> /etc/hosts;" >> /runscript.sh - -RUN echo "cd /controller/src/dlt/gateway; ./gradlew runServer" >> /runscript.sh -RUN chmod +x /runscript.sh - - -ENTRYPOINT ["sh", "/runscript.sh"] diff --git a/src/dlt/connector/Dockerfile b/src/dlt/connector/Dockerfile index d653bb217..51e9ec506 100644 --- a/src/dlt/connector/Dockerfile +++ b/src/dlt/connector/Dockerfile @@ -1,4 +1,18 @@ -FROM python:3-slim +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ @@ -14,25 +28,42 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ chmod +x /bin/grpc_health_probe # Get generic Python packages -RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools -# Set working directory +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto -# Create module sub-folders -RUN mkdir -p /var/teraflow/slice +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Get Python packages per module -COPY slice/requirements.in slice/requirements.in -RUN pip-compile --output-file=slice/requirements.txt slice/requirements.in -RUN python3 -m pip install -r slice/requirements.in +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/dlt/connector +WORKDIR /var/teraflow/dlt/connector +COPY src/dlt/connector/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt -# Add files into working directory -COPY common/. common -COPY context/. context -COPY interdomain/. interdomain -COPY service/. service -COPY slice/. slice +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ +COPY src/dlt/connector/. dlt/connector -# Start slice service -ENTRYPOINT ["python", "-m", "slice.service"] +# Start the service +ENTRYPOINT ["python", "-m", "dlt.connector.service"] diff --git a/src/dlt/connector/requirements.in b/src/dlt/connector/requirements.in index 162ecde82..e69de29bb 100644 --- a/src/dlt/connector/requirements.in +++ b/src/dlt/connector/requirements.in @@ -1,6 +0,0 @@ -grpcio==1.43.0 -grpcio-health-checking==1.43.0 -prometheus-client==0.13.0 -protobuf==3.19.3 -pytest==6.2.5 -pytest-benchmark==3.4.1 diff --git a/src/dlt/connector/tests/test_unitary.py b/src/dlt/connector/tests/test_unitary.py index f5f54798f..00c1164e1 100644 --- a/src/dlt/connector/tests/test_unitary.py +++ b/src/dlt/connector/tests/test_unitary.py @@ -18,8 +18,14 @@ from common.orm.Database import Database from common.message_broker.MessageBroker import MessageBroker from common.proto.context_pb2 import Context, ContextId, Device, DeviceId, Link, LinkId, Topology, TopologyId from context.client.ContextClient import ContextClient -from .PrepareTestScenario import context_db_mb, context_service, context_client # pylint: disable=unused-import -from .Objects import CONTEXTS, TOPOLOGIES, DEVICES, LINKS +from .PrepareTestScenario import ( + # pylint: disable=unused-import + dltgateway_service, + context_service_a, context_client_a, dltconnector_service_a, dltconnector_client_a, + context_service_b, context_client_b, dltconnector_service_b, dltconnector_client_b) +from .Objects import ( + DA_CONTEXTS, DA_TOPOLOGIES, DA_DEVICES, DA_LINKS, + DB_CONTEXTS, DB_TOPOLOGIES, DB_DEVICES, DB_LINKS) LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) @@ -27,9 +33,6 @@ LOGGER.setLevel(logging.DEBUG) def test_create_events( context_client : ContextClient, # pylint: disable=redefined-outer-name context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name - context_database = context_db_mb[0] - - context_database.clear_all() for context in CONTEXTS : context_client.SetContext (Context (**context )) for topology in TOPOLOGIES: context_client.SetTopology(Topology(**topology)) @@ -41,3 +44,11 @@ def test_create_events( for device in DEVICES : context_client.RemoveDevice (DeviceId (**device ['device_id' ])) for topology in TOPOLOGIES: context_client.RemoveTopology(TopologyId(**topology['topology_id'])) for context in CONTEXTS : context_client.RemoveContext (ContextId (**context ['context_id' ])) + + + + dltgateway_client = DltGatewayClient(host='127.0.0.1', port=50051) + dltgateway_collector = DltEventsCollector(dltgateway_client, log_events_received=True) + dltgateway_collector.start() + + dltgateway_collector.stop() diff --git a/src/dlt/gateway/Dockerfile b/src/dlt/gateway/Dockerfile new file mode 100644 index 000000000..92ef8e425 --- /dev/null +++ b/src/dlt/gateway/Dockerfile @@ -0,0 +1,41 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM zenika/kotlin:1.4-jdk12 + +# Make working directory move to it and copy DLT Gateway code +RUN mkdir -p /var/teraflow/dlt/gateway +WORKDIR /var/teraflow/dlt/gateway +COPY src/dlt/gateway/. ./ + +# Make directory for proto files and copy them +RUN mkdir proto +COPY proto/*.proto ./proto/ + +# Build DLT Gateway +RUN ./gradlew build + +EXPOSE 50051 + +# Create entrypoint.sh script +RUN echo "#!/bin/sh" > /entrypoint.sh +RUN echo "echo 195.37.154.24 peer0.org1.example.com >> /etc/hosts" >> /entrypoint.sh +RUN echo "echo 195.37.154.24 peer0.org2.example.com >> /etc/hosts" >> /entrypoint.sh +RUN echo "echo 195.37.154.24 orderer0.example.com >> /etc/hosts" >> /entrypoint.sh +RUN echo "cd /var/teraflow/dlt/gateway" >> /entrypoint.sh +RUN echo "./gradlew runServer" >> /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Gateway entry point +ENTRYPOINT ["sh", "/entrypoint.sh"] diff --git a/src/dlt/gateway/build.gradle.kts b/src/dlt/gateway/build.gradle.kts index c7213fbf7..b65aff89e 100644 --- a/src/dlt/gateway/build.gradle.kts +++ b/src/dlt/gateway/build.gradle.kts @@ -107,7 +107,7 @@ task("runServer", JavaExec::class) { sourceSets { main { proto { - srcDir("../../../proto") + srcDir("proto") srcDir("src/main/kotlin/proto") } } diff --git a/src/dlt/mock_blockchain/Dockerfile b/src/dlt/mock_blockchain/Dockerfile new file mode 100644 index 000000000..22199b5f8 --- /dev/null +++ b/src/dlt/mock_blockchain/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/mock_blockchain +WORKDIR /var/teraflow/mock_blockchain +COPY src/dlt/mock_blockchain/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/dlt/mock_blockchain/. mock_blockchain + +# Start the service +ENTRYPOINT ["python", "-m", "mock_blockchain.service"] diff --git a/src/dlt/mock_blockchain/__init__.py b/src/dlt/mock_blockchain/__init__.py new file mode 100644 index 000000000..9953c8205 --- /dev/null +++ b/src/dlt/mock_blockchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/dlt/mock_blockchain/requirements.in b/src/dlt/mock_blockchain/requirements.in new file mode 100644 index 000000000..e69de29bb diff --git a/src/dlt/mock_blockchain/service/__init__.py b/src/dlt/mock_blockchain/service/__init__.py new file mode 100644 index 000000000..9953c8205 --- /dev/null +++ b/src/dlt/mock_blockchain/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/dlt/mock_blockchain/service/__main__.py b/src/dlt/mock_blockchain/service/__main__.py new file mode 100644 index 000000000..359c6990a --- /dev/null +++ b/src/dlt/mock_blockchain/service/__main__.py @@ -0,0 +1,61 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Constants import ServiceNameEnum +from common.Settings import get_log_level, get_service_port_grpc +from common.proto.dlt_gateway_pb2_grpc import add_DltGatewayServiceServicer_to_server +from common.tests.MockServicerImpl_DltGateway import MockServicerImpl_DltGateway +from common.tools.service.GenericGrpcService import GenericGrpcService + +terminate = threading.Event() + +logging.basicConfig(level=get_log_level()) +LOGGER = logging.getLogger(__name__) + +class MockDltGatewayService(GenericGrpcService): + def __init__(self, cls_name: str = 'MockDltGatewayService') -> None: + port = get_service_port_grpc(ServiceNameEnum.DLT_GATEWAY) + super().__init__(port, cls_name=cls_name) + self.dltgateway_servicer = MockServicerImpl_DltGateway() + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + add_DltGatewayServiceServicer_to_server(self.dltgateway_servicer, self.server) + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Starting Mock DLT gateway service + grpc_service = MockDltGatewayService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=0.1): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) -- GitLab From 06c21622b81a2a886bf01be6c64c5983245b6356 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 27 Oct 2022 16:07:40 +0000 Subject: [PATCH 33/36] Updated deploy script: - Corrected URLs for multi-container pods - Added support for DLT component - Corrected minor errors with service/deployment naming --- deploy.sh | 96 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/deploy.sh b/deploy.sh index b0bd37442..add41fa13 100755 --- a/deploy.sh +++ b/deploy.sh @@ -66,44 +66,71 @@ echo "export PYTHONPATH=${PYTHONPATH}" >> $ENV_VARS_SCRIPT for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." - IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG" - IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" + docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" elif [ "$COMPONENT" == "pathcomp" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" - docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG" + docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" - docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" # next command is redundant, but helpful to keep cache updated between rebuilds - docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" + docker build -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + elif [ "$COMPONENT" == "dlt" ]; then + BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" + docker build -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" + + BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" + docker build -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" else - docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" + docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" fi if [ -n "$TFS_REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$TFS_REGISTRY_IMAGE'..." if [ "$COMPONENT" == "pathcomp" ]; then - TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" - docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG" + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') - TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" - docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG" + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" + docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log" - docker push "$IMAGE_URL-frontend" > "$PUSH_LOG" + docker push "$IMAGE_URL" > "$PUSH_LOG" + + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" + docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log" - docker push "$IMAGE_URL-backend" > "$PUSH_LOG" + docker push "$IMAGE_URL" > "$PUSH_LOG" + elif [ "$COMPONENT" == "dlt" ]; then + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log" + docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" + + PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log" + docker push "$IMAGE_URL" > "$PUSH_LOG" + + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log" + docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" + + PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log" + docker push "$IMAGE_URL" > "$PUSH_LOG" else + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" - docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" + docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" @@ -117,33 +144,48 @@ for COMPONENT in $TFS_COMPONENTS; do if [ -n "$TFS_REGISTRY_IMAGE" ]; then # Registry is set if [ "$COMPONENT" == "pathcomp" ]; then + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) - sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST" + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) - sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST" - - sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" + elif [ "$COMPONENT" == "dlt" ]; then + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" + + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" - sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" fi + + sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" else # Registry is not set if [ "$COMPONENT" == "pathcomp" ]; then VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) - sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST" + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $COMPONENT-frontend:$TFS_IMAGE_TAG#g" "$MANIFEST" VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) - sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST" + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $COMPONENT-backend:$TFS_IMAGE_TAG#g" "$MANIFEST" + elif [ "$COMPONENT" == "dlt" ]; then + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $COMPONENT-connector:$TFS_IMAGE_TAG#g" "$MANIFEST" - sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $COMPONENT-gateway:$TFS_IMAGE_TAG#g" "$MANIFEST" else VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) - sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" - sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" + sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $COMPONENT:$TFS_IMAGE_TAG#g" "$MANIFEST" fi + + sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" fi # TODO: harmonize names of the monitoring component @@ -157,7 +199,7 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Collecting env-vars for '$COMPONENT' component..." - SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json) + SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME}service --namespace $TFS_K8S_NAMESPACE -o json) if [ -z "${SERVICE_DATA}" ]; then continue; fi # Env vars for service's host address @@ -189,6 +231,7 @@ for EXTRA_MANIFEST in $TFS_EXTRA_MANIFESTS; do kubectl --namespace $TFS_K8S_NAMESPACE apply -f $EXTRA_MANIFEST printf "\n" done +printf "\n" # By now, leave these controls here. Some component dependencies are not well handled. @@ -203,8 +246,9 @@ fi for COMPONENT in $TFS_COMPONENTS; do echo "Waiting for '$COMPONENT' component..." + COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl wait --namespace $TFS_K8S_NAMESPACE \ - --for='condition=available' --timeout=300s deployment/${COMPONENT}service + --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME}service printf "\n" done -- GitLab From 161368fc99e9a866841ceb63b3057300682ac17e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 27 Oct 2022 16:48:55 +0000 Subject: [PATCH 34/36] Common: - removed unneeded genproto.sh script --- genproto.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 genproto.sh diff --git a/genproto.sh b/genproto.sh deleted file mode 100755 index d7ffe50e6..000000000 --- a/genproto.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -eu - -mkdir -p src/common/proto -rm -rf src/common/proto/*.py -touch src/common/proto/__init__.py -python3 -m grpc_tools.protoc -I=./proto --python_out=src/common/proto/ --grpc_python_out=src/common/proto/ proto/*.proto -find src/common/proto -type f -iname *.py -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -- GitLab From 394586c39d9892fc71943f93217b060fd3d46875 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 27 Oct 2022 16:49:54 +0000 Subject: [PATCH 35/36] DLT component: - implemented preliminary version of CI/CD pipeline definition for DLT (still not integrated in pipeline) --- .gitlab-ci.yml | 1 + src/dlt/.gitlab-ci.yml | 184 +++++++++++++++++++++++++++++++ src/dlt/connector/.gitlab-ci.yml | 72 ------------ 3 files changed, 185 insertions(+), 72 deletions(-) create mode 100644 src/dlt/.gitlab-ci.yml delete mode 100644 src/dlt/connector/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3de792462..78d2cf776 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,3 +42,4 @@ include: #- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' + - local: '/src/dlt/.gitlab-ci.yml' diff --git a/src/dlt/.gitlab-ci.yml b/src/dlt/.gitlab-ci.yml new file mode 100644 index 000000000..3c2013f50 --- /dev/null +++ b/src/dlt/.gitlab-ci.yml @@ -0,0 +1,184 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build dlt: + variables: + IMAGE_NAME: 'dlt' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + # This first build tags the builder resulting image to prevent being removed by dangling image removal command + - docker build -t "${IMAGE_NAME}-gateway:$IMAGE_TAG" -f ./src/$IMAGE_NAME/gateway/Dockerfile . + - docker build -t "${IMAGE_NAME}-connector:$IMAGE_TAG" -f ./src/$IMAGE_NAME/connector/Dockerfile . + - docker tag "${IMAGE_NAME}-gateway:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" + - docker tag "${IMAGE_NAME}-connector:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-connector:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-connector:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/.gitlab-ci.yml + - src/$IMAGE_NAME/gateway/**/*.{kt,kts,proto,pem,json} + - src/$IMAGE_NAME/gateway/build.gradle.kts + - src/$IMAGE_NAME/gateway/Dockerfile + - src/$IMAGE_NAME/gateway/gradle.properties + - src/$IMAGE_NAME/gateway/gradlew + - src/$IMAGE_NAME/gateway/gradlew.bat + - src/$IMAGE_NAME/gateway/settings.gradle.kts + - src/$IMAGE_NAME/connector/**/*.{py,in,yml} + - src/$IMAGE_NAME/connector/Dockerfile + - src/$IMAGE_NAME/connector/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit test dlt-gateway: + variables: + IMAGE_NAME: 'dlt' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build dlt + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep ${IMAGE_NAME}-connector; then docker rm -f ${IMAGE_NAME}-connector; else echo "${IMAGE_NAME}-connector image is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-gateway; then docker rm -f ${IMAGE_NAME}-gateway; else echo "${IMAGE_NAME}-gateway image is not in the system"; fi + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" + #- docker run --name ${IMAGE_NAME}-gateway -d -p 50051:50051 -v "$PWD/src/${IMAGE_NAME}/gateway/tests:/opt/results" --network=teraflowbridge ${IMAGE_NAME}-gateway:${IMAGE_TAG} + - docker run --name ${IMAGE_NAME}-gateway -d -p 50051:50051 --network=teraflowbridge ${IMAGE_NAME}-gateway:${IMAGE_TAG} + - sleep 5 + - docker ps -a + - docker logs ${IMAGE_NAME}-gateway + #- docker exec -i ${IMAGE_NAME}-gateway bash -c "curl -0 -v -X POST -H 'Expect:' -H 'Content-Type:\ application/json' http://127.0.0.1:8081/dlt/api/v1/compRoute -d @/var/teraflow/tests/pc-req.json" + #- docker kill --signal=SIGUSR1 dlt-gateway + #- docker exec -i ${IMAGE_NAME}-gateway bash -c "gcovr" + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker logs ${IMAGE_NAME}-gateway + - docker rm -f ${IMAGE_NAME}-gateway + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/.gitlab-ci.yml + - src/$IMAGE_NAME/gateway/**/*.{kt,kts,proto,pem,json} + - src/$IMAGE_NAME/gateway/build.gradle.kts + - src/$IMAGE_NAME/gateway/Dockerfile + - src/$IMAGE_NAME/gateway/gradle.properties + - src/$IMAGE_NAME/gateway/gradlew + - src/$IMAGE_NAME/gateway/gradlew.bat + - src/$IMAGE_NAME/gateway/settings.gradle.kts + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + #artifacts: + # when: always + # reports: + # junit: src/$IMAGE_NAME/gateway/tests/${IMAGE_NAME}-gateway_report.xml + +# Apply unit test to the component +unit test dlt-connector: + variables: + IMAGE_NAME: 'dlt' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build dlt + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge --subnet=172.28.0.0/24 --gateway=172.28.0.254 teraflowbridge; fi + - if docker container ls | grep ${IMAGE_NAME}-connector; then docker rm -f ${IMAGE_NAME}-connector; else echo "${IMAGE_NAME}-connector image is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-gateway; then docker rm -f ${IMAGE_NAME}-gateway; else echo "${IMAGE_NAME}-gateway image is not in the system"; fi + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-connector:$IMAGE_TAG" + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG" + - docker run --name ${IMAGE_NAME}-gateway -d -p 50051:50051 -v "$PWD/src/${IMAGE_NAME}/gateway/tests:/opt/results" --network=teraflowbridge --ip 172.28.0.1 $CI_REGISTRY_IMAGE/${IMAGE_NAME}-gateway:$IMAGE_TAG + - sleep 1 + - docker run --name ${IMAGE_NAME}-connector -d -p 8080:8080 --env "DLT_GATEWAY_HOST=172.28.0.1" --env "DLT_GATEWAY_PORT=50051" -v "$PWD/src/${IMAGE_NAME}/connector/tests:/opt/results" --network=teraflowbridge --ip 172.28.0.2 $CI_REGISTRY_IMAGE/${IMAGE_NAME}-connector:$IMAGE_TAG + - sleep 5 + - docker ps -a + - docker logs ${IMAGE_NAME}-connector + - docker logs ${IMAGE_NAME}-gateway + - docker exec -i ${IMAGE_NAME}-connector bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/connector/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}-connector_report.xml" + - docker exec -i ${IMAGE_NAME}-connector bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker ps -a + - docker logs ${IMAGE_NAME}-connector + - docker logs ${IMAGE_NAME}-gateway + - docker rm -f ${IMAGE_NAME}-connector + - docker rm -f ${IMAGE_NAME}-gateway + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/.gitlab-ci.yml + - src/$IMAGE_NAME/gateway/**/*.{kt,kts,proto,pem,json} + - src/$IMAGE_NAME/gateway/build.gradle.kts + - src/$IMAGE_NAME/gateway/Dockerfile + - src/$IMAGE_NAME/gateway/gradle.properties + - src/$IMAGE_NAME/gateway/gradlew + - src/$IMAGE_NAME/gateway/gradlew.bat + - src/$IMAGE_NAME/gateway/settings.gradle.kts + - src/$IMAGE_NAME/connector/**/*.{py,in,yml} + - src/$IMAGE_NAME/connector/Dockerfile + - src/$IMAGE_NAME/connector/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/connector/tests/${IMAGE_NAME}-connector_report.xml + +# Deployment of the service in Kubernetes Cluster +deploy dlt: + variables: + IMAGE_NAME: 'dlt' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: deploy + needs: + - unit test dlt-gateway + - unit test dlt-connector + # - integ_test execute + script: + - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' + - kubectl version + - kubectl get all + - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" + - kubectl get all + # environment: + # name: test + # url: https://example.com + # kubernetes: + # namespace: test + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + when: manual + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + when: manual diff --git a/src/dlt/connector/.gitlab-ci.yml b/src/dlt/connector/.gitlab-ci.yml deleted file mode 100644 index 08c58ae4a..000000000 --- a/src/dlt/connector/.gitlab-ci.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build, tag, and push the Docker images to the GitLab Docker registry -build slice: - variables: - IMAGE_NAME: 'slice' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: build - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/ - - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - rules: - - changes: - - src/$IMAGE_NAME/** - - .gitlab-ci.yml - -# Pull, execute, and run unitary tests for the Docker image from the GitLab registry -unit_test slice: - variables: - IMAGE_NAME: 'slice' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build slice - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run -d -p 4040:4040 --name $IMAGE_NAME --network=teraflowbridge "$IMAGE_NAME:$IMAGE_TAG" - - docker ps -a - - sleep 5 - - docker ps -a - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py" - after_script: - - docker stop $IMAGE_NAME - - docker rm $IMAGE_NAME - rules: - - changes: - - src/$IMAGE_NAME/** - - .gitlab-ci.yml - -# Deployment of the service in Kubernetes Cluster -deploy slice: - stage: deploy - needs: - - build slice - - unit_test slice - - dependencies all - - integ_test execute - script: - - kubectl version - - kubectl get all - - kubectl apply -f "manifests/sliceservice.yaml" - - kubectl delete pods --selector app=sliceservice - - kubectl get all -- GitLab From b8501763011a28d611d066034d56bf577845dbe3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 27 Oct 2022 16:52:42 +0000 Subject: [PATCH 36/36] DLT component: - deactivated CI/CD pipeline for DLT. It was activated by mistake --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78d2cf776..dac76342a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,4 +42,4 @@ include: #- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' - - local: '/src/dlt/.gitlab-ci.yml' + #- local: '/src/dlt/.gitlab-ci.yml' -- GitLab