Async refactoring of the code. Initial (34f8bb97) · Commits · TFS / controller

src/common/method_wrappers/Decorator.py

+46 −0

Original line number	Diff line number	Diff line
		@@ -12,6 +12,20 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import grpc, json, logging, threading
		from enum import Enum
		from prettytable import PrettyTable
		@@ -235,3 +249,35 @@ def safe_and_metered_rpc_method(metrics_pool : MetricsPool, logger : logging.Log
		grpc_context.abort(grpc.StatusCode.INTERNAL, str(e))
		return inner_wrapper
		return outer_wrapper

		def safe_and_metered_rpc_method_async(metrics_pool: MetricsPool, logger: logging.Logger):
		def outer_wrapper(func):
		method_name = func.__name__
		metrics = metrics_pool.get_metrics(method_name)
		histogram_duration, counter_started, counter_completed, counter_failed = metrics

		async def inner_wrapper(self, request, grpc_context: grpc.aio.ServicerContext):
		counter_started.inc()
		try:
		logger.debug('{:s} request: {:s}'.format(method_name, grpc_message_to_json_string(request)))
		reply = await func(self, request, grpc_context)
		logger.debug('{:s} reply: {:s}'.format(method_name, grpc_message_to_json_string(reply)))
		counter_completed.inc()
		return reply
		except ServiceException as e: # pragma: no cover (ServiceException not thrown)
		if e.code not in [grpc.StatusCode.NOT_FOUND, grpc.StatusCode.ALREADY_EXISTS]:
		# Assume not found or already exists is just a condition, not an error
		logger.exception('{:s} exception'.format(method_name))
		counter_failed.inc()
		else:
		counter_completed.inc()
		await grpc_context.abort(e.code, e.details)
		except Exception as e: # pragma: no cover, pylint: disable=broad-except
		logger.exception('{:s} exception'.format(method_name))
		counter_failed.inc()
		await grpc_context.abort(grpc.StatusCode.INTERNAL, str(e))

		return inner_wrapper

		return outer_wrapper

src/common/tools/service/GenericGrpcServiceAsync.py

0 → 100644

+72 −0

Original line number	Diff line number	Diff line
		# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		from typing import Optional, Union
		import grpc
		import logging
		from concurrent import futures
		from grpc_health.v1.health import HealthServicer, OVERALL_HEALTH
		from grpc_health.v1.health_pb2 import HealthCheckResponse
		from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
		from common.Settings import get_grpc_bind_address, get_grpc_grace_period, get_grpc_max_workers

		class GenericGrpcServiceAsync:
		def __init__(
		self, bind_port: Union[str, int], bind_address: Optional[str] = None, max_workers: Optional[int] = None,
		grace_period: Optional[int] = None, enable_health_servicer: bool = True, cls_name: str = __name__
		) -> None:
		self.logger = logging.getLogger(cls_name)
		self.bind_port = bind_port
		self.bind_address = get_grpc_bind_address() if bind_address is None else bind_address
		self.max_workers = get_grpc_max_workers() if max_workers is None else max_workers
		self.grace_period = get_grpc_grace_period() if grace_period is None else grace_period
		self.enable_health_servicer = enable_health_servicer
		self.endpoint = None
		self.health_servicer = None
		self.pool = None
		self.server = None

		async def install_servicers(self):
		pass

		async def start(self):
		self.endpoint = '{:s}:{:s}'.format(str(self.bind_address), str(self.bind_port))
		self.logger.info('Starting Service (tentative endpoint: {:s}, max_workers: {:s})...'.format(
		str(self.endpoint), str(self.max_workers)))

		self.pool = futures.ThreadPoolExecutor(max_workers=self.max_workers)
		self.server = grpc.aio.server(self.pool)

		await self.install_servicers() # Ensure this is awaited

		if self.enable_health_servicer:
		self.health_servicer = HealthServicer(
		experimental_non_blocking=True, experimental_thread_pool=futures.ThreadPoolExecutor(max_workers=1))
		add_HealthServicer_to_server(self.health_servicer, self.server)

		self.bind_port = self.server.add_insecure_port(self.endpoint)
		self.endpoint = '{:s}:{:s}'.format(str(self.bind_address), str(self.bind_port))
		self.logger.info('Listening on {:s}...'.format(str(self.endpoint)))
		await self.server.start()
		if self.enable_health_servicer:
		self.health_servicer.set(OVERALL_HEALTH, HealthCheckResponse.SERVING)

		self.logger.debug('Service started')

		async def stop(self):
		self.logger.debug('Stopping service (grace period {:s} seconds)...'.format(str(self.grace_period)))
		if self.enable_health_servicer:
		self.health_servicer.enter_graceful_shutdown()
		await self.server.stop(self.grace_period)
		self.logger.debug('Service stopped')

src/dlt/connector/client/DltConnectorClient.py

+44 −26

Original line number	Diff line number	Diff line
		@@ -12,7 +12,12 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import grpc, logging
		# DltConnectorClient.py

		import grpc
		import logging
		import asyncio

		from common.Constants import ServiceNameEnum
		from common.Settings import get_service_host, get_service_port_grpc
		from common.proto.context_pb2 import Empty, TopologyId
		@@ -35,77 +40,90 @@ class DltConnectorClient:
		LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint))
		self.channel = None
		self.stub = None
		self.connect()
		LOGGER.debug('Channel created')
		#self.connect()
		#LOGGER.debug('Channel created')

		def connect(self):
		self.channel = grpc.insecure_channel(self.endpoint)
		async def connect(self):
		self.channel = grpc.aio.insecure_channel(self.endpoint)
		self.stub = DltConnectorServiceStub(self.channel)
		LOGGER.debug('Channel created')

		def close(self):
		if self.channel is not None: self.channel.close()
		async def close(self):
		if self.channel is not None:
		await self.channel.close()
		self.channel = None
		self.stub = None

		@RETRY_DECORATOR
		def RecordAll(self, request : TopologyId) -> Empty:
		async def RecordAll(self, request: TopologyId) -> Empty:
		LOGGER.debug('RecordAll request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAll(request)
		response = await self.stub.RecordAll(request)
		LOGGER.debug('RecordAll result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllDevices(self, request : TopologyId) -> Empty:
		async def RecordAllDevices(self, request: TopologyId) -> Empty:
		LOGGER.debug('RecordAllDevices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllDevices(request)
		response = await self.stub.RecordAllDevices(request)
		LOGGER.debug('RecordAllDevices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordDevice(self, request : DltDeviceId) -> Empty:
		LOGGER.debug('RecordDevice request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordDevice(request)
		# async def RecordDevice(self, request: DltDeviceId) -> Empty:
		# LOGGER.debug('RECORD_DEVICE request received: {:s}'.format(grpc_message_to_json_string(request)))

		# Simulate some asynchronous processing delay
		# await asyncio.sleep(2) # Simulates processing time

		# Create a dummy response (Empty message)
		# response = Empty()

		# LOGGER.debug('RECORD_DEVICE processing complete for request: {:s}'.format(grpc_message_to_json_string(request)))
		# return response
		async def RecordDevice(self, request: DltDeviceId) -> Empty:
		LOGGER.debug('RECORD_DEVICE request: {:s}'.format(grpc_message_to_json_string(request)))
		response = await self.stub.RecordDevice(request)
		LOGGER.debug('RecordDevice result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllLinks(self, request : TopologyId) -> Empty:
		async def RecordAllLinks(self, request: TopologyId) -> Empty:
		LOGGER.debug('RecordAllLinks request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllLinks(request)
		response = await self.stub.RecordAllLinks(request)
		LOGGER.debug('RecordAllLinks result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordLink(self, request : DltLinkId) -> Empty:
		async def RecordLink(self, request: DltLinkId) -> Empty:
		LOGGER.debug('RecordLink request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordLink(request)
		response = await self.stub.RecordLink(request)
		LOGGER.debug('RecordLink result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllServices(self, request : TopologyId) -> Empty:
		async def RecordAllServices(self, request: TopologyId) -> Empty:
		LOGGER.debug('RecordAllServices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllServices(request)
		response = await self.stub.RecordAllServices(request)
		LOGGER.debug('RecordAllServices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordService(self, request : DltServiceId) -> Empty:
		async def RecordService(self, request: DltServiceId) -> Empty:
		LOGGER.debug('RecordService request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordService(request)
		response = await self.stub.RecordService(request)
		LOGGER.debug('RecordService result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllSlices(self, request : TopologyId) -> Empty:
		async def RecordAllSlices(self, request: TopologyId) -> Empty:
		LOGGER.debug('RecordAllSlices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllSlices(request)
		response = await self.stub.RecordAllSlices(request)
		LOGGER.debug('RecordAllSlices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordSlice(self, request : DltSliceId) -> Empty:
		async def RecordSlice(self, request: DltSliceId) -> Empty:
		LOGGER.debug('RecordSlice request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordSlice(request)
		response = await self.stub.RecordSlice(request)
		LOGGER.debug('RecordSlice result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

src/dlt/connector/client/DltConnectorClientSync.py

0 → 100644

+111 −0

Original line number	Diff line number	Diff line
		# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import grpc, logging
		from common.Constants import ServiceNameEnum
		from common.Settings import get_service_host, get_service_port_grpc
		from common.proto.context_pb2 import Empty, TopologyId
		from common.proto.dlt_connector_pb2 import DltDeviceId, DltLinkId, DltServiceId, DltSliceId
		from common.proto.dlt_connector_pb2_grpc import DltConnectorServiceStub
		from common.tools.client.RetryDecorator import retry, delay_exponential
		from common.tools.grpc.Tools import grpc_message_to_json_string

		LOGGER = logging.getLogger(__name__)
		LOGGER.setLevel(logging.DEBUG)
		MAX_RETRIES = 15
		DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
		RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')

		class DltConnectorClientSync:
		def __init__(self, host=None, port=None):
		if not host: host = get_service_host(ServiceNameEnum.DLT)
		if not port: port = get_service_port_grpc(ServiceNameEnum.DLT)
		self.endpoint = '{:s}:{:s}'.format(str(host), str(port))
		LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint))
		self.channel = None
		self.stub = None
		self.connect()
		LOGGER.debug('Channel created')

		def connect(self):
		self.channel = grpc.insecure_channel(self.endpoint)
		self.stub = DltConnectorServiceStub(self.channel)

		def close(self):
		if self.channel is not None: self.channel.close()
		self.channel = None
		self.stub = None

		@RETRY_DECORATOR
		def RecordAll(self, request : TopologyId) -> Empty:
		LOGGER.debug('RecordAll request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAll(request)
		LOGGER.debug('RecordAll result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllDevices(self, request : TopologyId) -> Empty:
		LOGGER.debug('RecordAllDevices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllDevices(request)
		LOGGER.debug('RecordAllDevices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordDevice(self, request : DltDeviceId) -> Empty:
		LOGGER.debug('RecordDevice request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordDevice(request)
		LOGGER.debug('RecordDevice result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllLinks(self, request : TopologyId) -> Empty:
		LOGGER.debug('RecordAllLinks request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllLinks(request)
		LOGGER.debug('RecordAllLinks result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordLink(self, request : DltLinkId) -> Empty:
		LOGGER.debug('RecordLink request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordLink(request)
		LOGGER.debug('RecordLink result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllServices(self, request : TopologyId) -> Empty:
		LOGGER.debug('RecordAllServices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllServices(request)
		LOGGER.debug('RecordAllServices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordService(self, request : DltServiceId) -> Empty:
		LOGGER.debug('RecordService request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordService(request)
		LOGGER.debug('RecordService result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordAllSlices(self, request : TopologyId) -> Empty:
		LOGGER.debug('RecordAllSlices request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordAllSlices(request)
		LOGGER.debug('RecordAllSlices result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

		@RETRY_DECORATOR
		def RecordSlice(self, request : DltSliceId) -> Empty:
		LOGGER.debug('RecordSlice request: {:s}'.format(grpc_message_to_json_string(request)))
		response = self.stub.RecordSlice(request)
		LOGGER.debug('RecordSlice result: {:s}'.format(grpc_message_to_json_string(response)))
		return response

src/dlt/connector/client/DltEventsCollector.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@ from typing import Callable, Optional
		import grpc, logging, queue, threading, time
		from common.proto.dlt_gateway_pb2 import DltRecordEvent, DltRecordSubscription
		from common.tools.grpc.Tools import grpc_message_to_json_string
		from dlt.connector.client.DltGatewayClient import DltGatewayClient
		from dlt.connector.client.DltGatewayClientEvent import DltGatewayClientEvent

		LOGGER = logging.getLogger(__name__)
		LOGGER.setLevel(logging.DEBUG)
		@@ -31,7 +31,7 @@ LOGGER.setLevel(logging.DEBUG)

		class DltEventsCollector(threading.Thread):
		def __init__(
		self, dltgateway_client : DltGatewayClient,
		self, dltgateway_client : DltGatewayClientEvent,
		log_events_received : bool = False,
		event_handler : Optional[Callable[[DltRecordEvent], Optional[DltRecordEvent]]] = None,
		) -> None: