Merge branch 'feat/device-improve-perf-metrics-per-driver' into 'develop' (03d7ade6) · Commits · TFS / controller

deploy/tfs.sh

+3 −3

Original line number	Diff line number	Diff line
		@@ -486,11 +486,11 @@ if [[ "$TFS_COMPONENTS" == "webui" ]]; then
		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
		echo

		# Dashboard: Device ConfigureDevice Details
		curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_prom_device_config_exec_details.json' \
		# Dashboard: Device Execution Details
		curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_prom_device_exec_details.json' \
		${GRAFANA_URL_UPDATED}/api/dashboards/db
		echo
		DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-dev-confdev"
		DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-dev-exec"
		DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')
		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
		echo

src/device/service/DeviceServiceServicerImpl.py

+70 −11

Original line number	Diff line number	Diff line
		@@ -37,8 +37,8 @@ LOGGER = logging.getLogger(__name__)

		METRICS_POOL = MetricsPool('Device', 'RPC')

		METRICS_POOL_DETAILS = MetricsPool('Device', 'exec_details', labels={
		'step_name': '',
		METRICS_POOL_DETAILS = MetricsPool('Device', 'execution', labels={
		'driver': '', 'operation': '', 'step': '',
		})

		class DeviceServiceServicerImpl(DeviceServiceServicer):
		@@ -51,11 +51,15 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):

		@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
		def AddDevice(self, request : Device, context : grpc.ServicerContext) -> DeviceId:
		t0 = time.time()

		device_uuid = request.device_id.device_uuid.uuid

		connection_config_rules = check_connect_rules(request.device_config)
		check_no_endpoints(request.device_endpoints)

		t1 = time.time()

		context_client = ContextClient()
		device = get_device(context_client, device_uuid, rw_copy=True)
		if device is None:
		@@ -73,10 +77,15 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
		# update device_uuid to honor UUID provided by Context
		device_uuid = device.device_id.device_uuid.uuid

		t2 = time.time()

		self.mutex_queues.wait_my_turn(device_uuid)
		t3 = time.time()
		try:
		driver : _Driver = get_driver(self.driver_instance_cache, device)

		t4 = time.time()

		errors = []

		# Sub-devices and sub-links are exposed by intermediate controllers or represent mgmt links.
		@@ -86,13 +95,23 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
		new_sub_links : Dict[str, Link] = dict()

		if len(device.device_endpoints) == 0:
		t5 = time.time()
		# created from request, populate endpoints using driver
		errors.extend(populate_endpoints(
		device, driver, self.monitoring_loops, new_sub_devices, new_sub_links))
		t6 = time.time()
		t_pop_endpoints = t6 - t5
		else:
		t_pop_endpoints = None

		if len(device.device_config.config_rules) == len(connection_config_rules):
		# created from request, populate config rules using driver
		t7 = time.time()
		errors.extend(populate_config_rules(device, driver))
		t8 = time.time()
		t_pop_config_rules = t8 - t7
		else:
		t_pop_config_rules = None

		# TODO: populate components

		@@ -100,22 +119,60 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
		for error in errors: LOGGER.error(error)
		raise OperationFailedException('AddDevice', extra_details=errors)

		t9 = time.time()

		device.device_operational_status = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_DISABLED
		device_id = context_client.SetDevice(device)

		t10 = time.time()

		for sub_device in new_sub_devices.values():
		context_client.SetDevice(sub_device)

		t11 = time.time()

		for sub_links in new_sub_links.values():
		context_client.SetLink(sub_links)

		t12 = time.time()

		# Update endpoint monitoring resources with UUIDs
		device_with_uuids = get_device(
		context_client, device_id.device_uuid.uuid, rw_copy=False, include_endpoints=True,
		include_components=False, include_config_rules=False)
		populate_endpoint_monitoring_resources(device_with_uuids, self.monitoring_loops)

		t13 = time.time()

		context_client.close()

		t14 = time.time()

		metrics_labels = dict(driver=driver.name, operation='add_device')

		histogram_duration : Histogram = METRICS_POOL_DETAILS.get_or_create(
		'details', MetricTypeEnum.HISTOGRAM_DURATION)
		histogram_duration.labels(step='total' , **metrics_labels).observe(t14-t0)
		histogram_duration.labels(step='execution' , **metrics_labels).observe(t14-t3)
		histogram_duration.labels(step='endpoint_checks' , **metrics_labels).observe(t1-t0)
		histogram_duration.labels(step='get_device' , **metrics_labels).observe(t2-t1)
		histogram_duration.labels(step='wait_queue' , **metrics_labels).observe(t3-t2)
		histogram_duration.labels(step='get_driver' , **metrics_labels).observe(t4-t3)
		histogram_duration.labels(step='set_device' , **metrics_labels).observe(t10-t9)
		histogram_duration.labels(step='populate_monit_rsrc', **metrics_labels).observe(t13-t12)

		if t_pop_endpoints is not None:
		histogram_duration.labels(step='populate_endpoints', **metrics_labels).observe(t_pop_endpoints)

		if t_pop_config_rules is not None:
		histogram_duration.labels(step='populate_config_rules', **metrics_labels).observe(t_pop_config_rules)

		if len(new_sub_devices) > 0:
		histogram_duration.labels(step='set_sub_devices', **metrics_labels).observe(t11-t10)

		if len(new_sub_links) > 0:
		histogram_duration.labels(step='set_sub_links', **metrics_labels).observe(t12-t11)

		return device_id
		finally:
		self.mutex_queues.signal_done(device_uuid)
		@@ -195,16 +252,18 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):

		t9 = time.time()

		metrics_labels = dict(driver=driver.name, operation='configure_device')

		histogram_duration : Histogram = METRICS_POOL_DETAILS.get_or_create(
		'ConfigureDevice', MetricTypeEnum.HISTOGRAM_DURATION)
		histogram_duration.labels(step_name='total' ).observe(t9-t0)
		histogram_duration.labels(step_name='wait_queue' ).observe(t1-t0)
		histogram_duration.labels(step_name='execution' ).observe(t9-t1)
		histogram_duration.labels(step_name='get_device' ).observe(t3-t2)
		histogram_duration.labels(step_name='split_rules' ).observe(t5-t4)
		histogram_duration.labels(step_name='configure_rules' ).observe(t6-t5)
		histogram_duration.labels(step_name='deconfigure_rules').observe(t7-t6)
		histogram_duration.labels(step_name='set_device' ).observe(t9-t8)
		'details', MetricTypeEnum.HISTOGRAM_DURATION)
		histogram_duration.labels(step='total' , **metrics_labels).observe(t9-t0)
		histogram_duration.labels(step='wait_queue' , **metrics_labels).observe(t1-t0)
		histogram_duration.labels(step='execution' , **metrics_labels).observe(t9-t1)
		histogram_duration.labels(step='get_device' , **metrics_labels).observe(t3-t2)
		histogram_duration.labels(step='split_rules' , **metrics_labels).observe(t5-t4)
		histogram_duration.labels(step='configure_rules' , **metrics_labels).observe(t6-t5)
		histogram_duration.labels(step='deconfigure_rules', **metrics_labels).observe(t7-t6)
		histogram_duration.labels(step='set_device' , **metrics_labels).observe(t9-t8)

		return device_id
		finally:

src/device/service/driver_api/_Driver.py

+17 −2

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@ RESOURCE_ACL = '__acl__'


		class _Driver:
		def __init__(self, address: str, port: int, **settings) -> None:
		def __init__(self, name : str, address: str, port: int, **settings) -> None:
		""" Initialize Driver.
		Parameters:
		address : str
		@@ -37,7 +37,22 @@ class _Driver:
		**settings
		Extra settings required by the driver.
		"""
		raise NotImplementedError()
		self._name = name
		self._address = address
		self._port = port
		self._settings = settings

		@property
		def name(self): return self._name

		@property
		def address(self): return self._address

		@property
		def port(self): return self._port

		@property
		def settings(self): return self._settings

		def Connect(self) -> bool:
		""" Connect to the Device.

src/device/service/drivers/emulated/EmulatedDriver.py

+5 −3

Original line number	Diff line number	Diff line
		@@ -31,16 +31,18 @@ LOGGER = logging.getLogger(__name__)

		RE_GET_ENDPOINT_FROM_INTERFACE = re.compile(r'^\/interface\[([^\]]+)\].*')

		METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'emulated'})
		DRIVER_NAME = 'emulated'
		METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME})

		class EmulatedDriver(_Driver):
		def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called
		def __init__(self, address : str, port : int, **settings) -> None:
		super().__init__(DRIVER_NAME, address, port, **settings)
		self.__lock = threading.Lock()
		self.__initial = TreeNode('.')
		self.__running = TreeNode('.')
		self.__subscriptions = TreeNode('.')

		endpoints = settings.get('endpoints', [])
		endpoints = self.settings.get('endpoints', [])
		endpoint_resources = []
		for endpoint in endpoints:
		endpoint_resource = compose_resource_endpoint(endpoint)

src/device/service/drivers/ietf_l2vpn/IetfL2VpnDriver.py

+9 −7

Original line number	Diff line number	Diff line
		@@ -39,21 +39,23 @@ ALL_RESOURCE_KEYS = [

		SERVICE_TYPE = 'ELINE'

		METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'ietf_l2vpn'})
		DRIVER_NAME = 'ietf_l2vpn'
		METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME})

		class IetfL2VpnDriver(_Driver):
		def __init__(self, address: str, port: int, **settings) -> None: # pylint: disable=super-init-not-called
		def __init__(self, address: str, port: int, **settings) -> None:
		super().__init__(DRIVER_NAME, address, port, **settings)
		self.__lock = threading.Lock()
		self.__started = threading.Event()
		self.__terminate = threading.Event()
		username = settings.get('username')
		password = settings.get('password')
		scheme = settings.get('scheme', 'http')
		wim = {'wim_url': '{:s}://{:s}:{:d}'.format(scheme, address, int(port))}
		username = self.settings.get('username')
		password = self.settings.get('password')
		scheme = self.settings.get('scheme', 'http')
		wim = {'wim_url': '{:s}://{:s}:{:d}'.format(scheme, self.address, int(self.port))}
		wim_account = {'user': username, 'password': password}
		# Mapping updated dynamically with each request
		config = {'mapping_not_needed': False, 'service_endpoint_mapping': []}
		self.dac = TfsDebugApiClient(address, int(port), scheme=scheme, username=username, password=password)
		self.dac = TfsDebugApiClient(self.address, int(self.port), scheme=scheme, username=username, password=password)
		self.wim = WimconnectorIETFL2VPN(wim, wim_account, config=config)
		self.conn_info = {} # internal database emulating OSM storage provided to WIM Connectors