Merge branch 'feat/device-perf-eval' of... (07276892) · Commits · TFS / controller

src/device/service/drivers/microwave/IETFApiDriver.py

+15 −5

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@
		# limitations under the License.

		import logging, requests, threading
		from requests.auth import HTTPBasicAuth
		from typing import Any, Iterator, List, Optional, Tuple, Union
		from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method
		from common.type_checkers.Checkers import chk_string, chk_type
		@@ -29,15 +30,20 @@ class IETFApiDriver(_Driver):
		self.__lock = threading.Lock()
		self.__started = threading.Event()
		self.__terminate = threading.Event()
		self.__ietf_root = 'https://' + address + ':' + str(port)
		username = settings.get('username')
		password = settings.get('password')
		self.__auth = HTTPBasicAuth(username, password) if username is not None and password is not None else None
		scheme = settings.get('scheme', 'http')
		self.__ietf_root = '{:s}://{:s}:{:d}'.format(scheme, address, int(port))
		self.__timeout = int(settings.get('timeout', 120))
		self.__node_ids = set(settings.get('node_ids', []))

		def Connect(self) -> bool:
		url = self.__ietf_root + '/nmswebs/restconf/data/ietf-network:networks'
		with self.__lock:
		if self.__started.is_set(): return True
		try:
		requests.get(url, timeout=self.__timeout, verify=False)
		requests.get(url, timeout=self.__timeout, verify=False, auth=self.__auth)
		except requests.exceptions.Timeout:
		LOGGER.exception('Timeout connecting {:s}'.format(str(self.__ietf_root)))
		return False
		@@ -67,7 +73,9 @@ class IETFApiDriver(_Driver):
		for i, resource_key in enumerate(resource_keys):
		str_resource_name = 'resource_key[#{:d}]'.format(i)
		chk_string(str_resource_name, resource_key, allow_empty=False)
		results.extend(config_getter(self.__ietf_root, resource_key, self.__timeout))
		results.extend(config_getter(
		self.__ietf_root, resource_key, timeout=self.__timeout, auth=self.__auth,
		node_ids=self.__node_ids))
		return results

		@metered_subclass_method(METRICS_POOL)
		@@ -87,7 +95,8 @@ class IETFApiDriver(_Driver):
		uuid = find_key(resource, 'uuid')

		data = create_connectivity_service(
		self.__ietf_root, self.__timeout, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id)
		self.__ietf_root, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id,
		timeout=self.__timeout, auth=self.__auth)
		results.extend(data)
		return results

		@@ -99,7 +108,8 @@ class IETFApiDriver(_Driver):
		for resource in resources:
		LOGGER.info('resource = {:s}'.format(str(resource)))
		uuid = find_key(resource, 'uuid')
		results.extend(delete_connectivity_service(self.__ietf_root, self.__timeout, uuid))
		results.extend(delete_connectivity_service(
		self.__ietf_root, uuid, timeout=self.__timeout, auth=self.__auth))
		return results

		@metered_subclass_method(METRICS_POOL)

src/device/service/drivers/microwave/Tools.py

+46 −36

Original line number	Diff line number	Diff line
		@@ -13,6 +13,8 @@
		# limitations under the License.

		import json, logging, requests
		from requests.auth import HTTPBasicAuth
		from typing import Optional, Set
		from device.service.driver_api._Driver import RESOURCE_ENDPOINTS

		LOGGER = logging.getLogger(__name__)
		@@ -28,6 +30,8 @@ def find_key(resource, key):
		return json.loads(resource[1])[key]

		# this function exports only the endpoints which are not already involved in a microwave physical link
		# TODO: improvement: create a Set[Tuple[node_id:str, tp_id:str]] containing the endpoints involved in links
		# TODO: exportable endpoints are those not in this set. That will prevent looping through links for every endpoint
		def is_exportable_endpoint(node, termination_point_id, links):
		# for each link we check if the endpoint (termination_point_id) is already used by an existing link
		for link in links:
		@@ -39,7 +43,10 @@ def is_exportable_endpoint(node, termination_point_id, links):
		return False
		return True

		def config_getter(root_url, resource_key, timeout):
		def config_getter(
		root_url : str, resource_key : str, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None,
		node_ids : Set[str] = set()
		):
		# getting endpoints
		network_name = 'SIAE-ETH-TOPOLOGY'
		FIELDS = ''.join([
		@@ -51,51 +58,53 @@ def config_getter(root_url, resource_key, timeout):
		url = URL_TEMPLATE.format(root_url, network_name, FIELDS)

		result = []
		if resource_key == RESOURCE_ENDPOINTS:
		# getting existing endpoints
		try:
		response = requests.get(url, timeout=timeout, verify=False)
		except requests.exceptions.Timeout:
		LOGGER.exception('Timeout connecting {:s}'.format(url))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception retrieving {:s}'.format(resource_key))
		result.append((resource_key, e))
		else:
		response = requests.get(url, timeout=timeout, verify=False, auth=auth)
		context = json.loads(response.content)
		if resource_key == RESOURCE_ENDPOINTS:
		network_instance = context.get('ietf-network:network', {})
		links = network_instance.get('ietf-network-topology:link', [])
		for sip in network_instance['node']:
		tp = sip['ietf-network-topology:termination-point']
		node_id = sip['node-id']
		for te in tp:
		tp_id = te['tp-id']
		for node in network_instance['node']:
		node_id = node['node-id']
		if len(node_ids) > 0 and node_id not in node_ids: continue
		tp_list = node['ietf-network-topology:termination-point']
		for tp in tp_list:
		tp_id = tp['tp-id']
		if not is_exportable_endpoint(node_id, tp_id, links): continue
		resource_key = '/endpoints/endpoint[{:s}:{:s}]'.format(node_id,tp_id)
		resource_value = {'uuid': tp_id, 'type': te['ietf-te-topology:te']['name']}
		tp_uuid = '{:s}:{:s}'.format(node_id,tp_id)
		resource_key = '/endpoints/endpoint[{:s}]'.format(tp_uuid)
		resource_value = {'uuid': tp_uuid, 'type': tp['ietf-te-topology:te']['name']}
		result.append((resource_key, resource_value))

		# getting created services
		url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)
		try:
		response = requests.get(url, timeout=timeout, verify=False)
		except requests.exceptions.Timeout:
		LOGGER.exception('Timeout connecting {:s}'.format(url))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception retrieving {:s}'.format(resource_key))
		LOGGER.exception('Exception retrieving/parsing endpoints for {:s}'.format(resource_key))
		result.append((resource_key, e))
		else:
		# getting created services
		url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)
		try:
		response = requests.get(url, timeout=timeout, verify=False, auth=auth)
		context = json.loads(response.content)
		if resource_key == RESOURCE_ENDPOINTS:
		etht_service = context.get('ietf-eth-tran-service:etht-svc', {})
		service_instances = etht_service.get('etht-svc-instances', [])
		for service in service_instances:
		service_name = service['etht-svc-name']
		resource_key = '/services/service[{:s}]'.format(service_name)
		resource_value = {'uuid': service_name, 'type': service['etht-svc-type']}
		result.append((resource_key, resource_value))
		result.append((resource_key, service))
		except requests.exceptions.Timeout:
		LOGGER.exception('Timeout connecting {:s}'.format(url))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception retrieving/parsing services for {:s}'.format(resource_key))
		result.append((resource_key, e))

		return result

		def create_connectivity_service(
		root_url, timeout, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id):
		root_url, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id,
		auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None
		):

		url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)
		headers = {'content-type': 'application/json'}
		@@ -128,7 +137,8 @@ def create_connectivity_service(
		results = []
		try:
		LOGGER.info('Connectivity service {:s}: {:s}'.format(str(uuid), str(data)))
		response = requests.post(url=url, data=json.dumps(data), timeout=timeout, headers=headers, verify=False)
		response = requests.post(
		url=url, data=json.dumps(data), timeout=timeout, headers=headers, verify=False, auth=auth)
		LOGGER.info('Microwave Driver response: {:s}'.format(str(response)))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception creating ConnectivityService(uuid={:s}, data={:s})'.format(str(uuid), str(data)))
		@@ -140,12 +150,12 @@ def create_connectivity_service(
		results.append(response.status_code in HTTP_OK_CODES)
		return results

		def delete_connectivity_service(root_url, timeout, uuid):
		def delete_connectivity_service(root_url, uuid, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None):
		url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc/etht-svc-instances={:s}'
		url = url.format(root_url, uuid)
		results = []
		try:
		response = requests.delete(url=url, timeout=timeout, verify=False)
		response = requests.delete(url=url, timeout=timeout, verify=False, auth=auth)
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception deleting ConnectivityService(uuid={:s})'.format(str(uuid)))
		results.append(e)

src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py

+5 −3

Original line number	Diff line number	Diff line
		@@ -81,7 +81,7 @@ class MicrowaveServiceHandler(_ServiceHandler):

		device_uuid = endpoints[0][0]
		device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid)))
		json_config_rule = json_config_rule_set('/service[{:s}]'.format(service_uuid), {
		json_config_rule = json_config_rule_set('/services/service[{:s}]'.format(service_uuid), {
		'uuid' : service_uuid,
		'node_id_src': node_id_src,
		'tp_id_src' : tp_id_src,
		@@ -111,11 +111,13 @@ class MicrowaveServiceHandler(_ServiceHandler):
		results = []
		try:
		chk_type('endpoints', endpoints, list)
		if len(endpoints) != 2: raise Exception('len(endpoints) != 2')
		if len(endpoints) < 1: raise Exception('len(endpoints) < 1')

		device_uuid = endpoints[0][0]
		device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid)))
		json_config_rule = json_config_rule_delete('/service[{:s}]'.format(service_uuid), {'uuid': service_uuid})
		json_config_rule = json_config_rule_delete('/services/service[{:s}]'.format(service_uuid), {
		'uuid': service_uuid
		})
		del device.device_config.config_rules[:]
		device.device_config.config_rules.append(ConfigRule(**json_config_rule))
		self.__task_executor.configure_device(device)

src/tests/tools/load_gen/Constants.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ class RequestType(Enum):
		SERVICE_L2NM = 'svc-l2nm'
		SERVICE_L3NM = 'svc-l3nm'
		SERVICE_TAPI = 'svc-tapi'
		SERVICE_MW = 'svc-mw'
		SLICE_L2NM = 'slc-l2nm'
		SLICE_L3NM = 'slc-l3nm'

src/tests/tools/load_gen/RequestGenerator.py

+36 −13

Original line number	Diff line number	Diff line
		@@ -109,7 +109,8 @@ class RequestGenerator:
		LOGGER.info('[dump_state] used_device_endpoints = {:s}'.format(json.dumps(self._used_device_endpoints)))

		def _use_device_endpoint(
		self, service_uuid : str, endpoint_types : Optional[Set[str]] = None, exclude_device_uuids : Set[str] = set()
		self, service_uuid : str, request_type : RequestType, endpoint_types : Optional[Set[str]] = None,
		exclude_device_uuids : Set[str] = set(), exclude_endpoint_uuids : Set[Tuple[str, str]] = set(),
		) -> Optional[Tuple[str, str]]:
		with self._lock:
		compatible_endpoints : Set[Tuple[str, str]] = set()
		@@ -118,9 +119,14 @@ class RequestGenerator:
		if endpoint_types is None:
		# allow all
		elegible_device_endpoints : Dict[str, Set[str]] = {
		device_uuid:device_endpoint_uuids
		device_uuid:[
		endpoint_uuid for endpoint_uuid in device_endpoint_uuids
		if (len(exclude_endpoint_uuids) == 0) or \
		((device_uuid,endpoint_uuid) not in exclude_endpoint_uuids)
		]
		for device_uuid,device_endpoint_uuids in self._available_device_endpoints.items()
		if device_uuid not in exclude_device_uuids and len(device_endpoint_uuids) > 0
		if (device_uuid not in exclude_device_uuids) and \
		(len(device_endpoint_uuids) > 0)
		}
		else:
		# allow only compatible endpoints
		@@ -132,6 +138,7 @@ class RequestGenerator:
		if device_uuid in exclude_device_uuids or len(device_endpoint_uuids) == 0: continue
		for endpoint_uuid in device_endpoint_uuids:
		endpoint_key = (device_uuid,endpoint_uuid)
		if endpoint_key in exclude_endpoint_uuids: continue
		if endpoint_key not in compatible_endpoints: continue
		elegible_device_endpoints.setdefault(device_uuid, set()).add(endpoint_uuid)

		@@ -145,16 +152,19 @@ class RequestGenerator:
		'compatible_endpoints={:s}'.format(str(compatible_endpoints)),
		]))
		return None

		device_uuid = random.choice(list(elegible_device_endpoints.keys()))
		device_endpoint_uuids = elegible_device_endpoints.get(device_uuid)
		endpoint_uuid = random.choice(list(device_endpoint_uuids))
		if request_type not in {RequestType.SERVICE_MW}:
		# reserve the resources
		self._available_device_endpoints.setdefault(device_uuid, set()).discard(endpoint_uuid)
		self._used_device_endpoints.setdefault(device_uuid, dict())[endpoint_uuid] = service_uuid
		return device_uuid, endpoint_uuid

		def _release_device_endpoint(self, device_uuid : str, endpoint_uuid : str) -> None:
		with self._lock:
		self._used_device_endpoints.setdefault(device_uuid, set()).pop(endpoint_uuid, None)
		self._used_device_endpoints.setdefault(device_uuid, dict()).pop(endpoint_uuid, None)
		self._available_device_endpoints.setdefault(device_uuid, set()).add(endpoint_uuid)

		def compose_request(self) -> Optional[Dict]:
		@@ -168,7 +178,9 @@ class RequestGenerator:
		# choose request type
		request_type = random.choice(self._parameters.request_types)

		if request_type in {RequestType.SERVICE_L2NM, RequestType.SERVICE_L3NM, RequestType.SERVICE_TAPI}:
		if request_type in {
		RequestType.SERVICE_L2NM, RequestType.SERVICE_L3NM, RequestType.SERVICE_TAPI, RequestType.SERVICE_MW
		}:
		return self._compose_service(num_request, request_uuid, request_type)
		elif request_type in {RequestType.SLICE_L2NM, RequestType.SLICE_L3NM}:
		return self._compose_slice(num_request, request_uuid, request_type)
		@@ -176,7 +188,7 @@ class RequestGenerator:
		def _compose_service(self, num_request : int, request_uuid : str, request_type : str) -> Optional[Dict]:
		# choose source endpoint
		src_endpoint_types = set(ENDPOINT_COMPATIBILITY.keys()) if request_type in {RequestType.SERVICE_TAPI} else None
		src = self._use_device_endpoint(request_uuid, endpoint_types=src_endpoint_types)
		src = self._use_device_endpoint(request_uuid, request_type, endpoint_types=src_endpoint_types)
		if src is None:
		LOGGER.warning('>> No source endpoint is available')
		return None
		@@ -188,11 +200,12 @@ class RequestGenerator:
		dst_endpoint_types = {dst_endpoint_type} if request_type in {RequestType.SERVICE_TAPI} else None

		# identify excluded destination devices
		exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI} else {src_device_uuid}
		exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI, RequestType.SERVICE_MW} else {src_device_uuid}

		# choose feasible destination endpoint
		dst = self._use_device_endpoint(
		request_uuid, endpoint_types=dst_endpoint_types, exclude_device_uuids=exclude_device_uuids)
		request_uuid, request_type, endpoint_types=dst_endpoint_types, exclude_device_uuids=exclude_device_uuids,
		exclude_endpoint_uuids={src})

		# if destination endpoint not found, release source, and terminate current service generation
		if dst is None:
		@@ -310,19 +323,29 @@ class RequestGenerator:
		return json_service_tapi_planned(
		request_uuid, endpoint_ids=endpoint_ids, constraints=[], config_rules=config_rules)

		elif request_type == RequestType.SERVICE_MW:
		vlan_id = 1000 + num_request % 1000
		config_rules = [
		json_config_rule_set('/settings', {
		'vlan_id': vlan_id,
		}),
		]
		return json_service_l2nm_planned(
		request_uuid, endpoint_ids=endpoint_ids, constraints=[], config_rules=config_rules)

		def _compose_slice(self, num_request : int, request_uuid : str, request_type : str) -> Optional[Dict]:
		# choose source endpoint
		src = self._use_device_endpoint(request_uuid)
		src = self._use_device_endpoint(request_uuid, request_type)
		if src is None:
		LOGGER.warning('>> No source endpoint is available')
		return None
		src_device_uuid,src_endpoint_uuid = src

		# identify excluded destination devices
		exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI} else {src_device_uuid}
		exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI, RequestType.SERVICE_MW} else {src_device_uuid}

		# choose feasible destination endpoint
		dst = self._use_device_endpoint(request_uuid, exclude_device_uuids=exclude_device_uuids)
		dst = self._use_device_endpoint(request_uuid, request_type, exclude_device_uuids=exclude_device_uuids)

		# if destination endpoint not found, release source, and terminate current service generation
		if dst is None: