Merge branch 'perf/tid-openconfig' into 'develop' (41e52b80) · Commits · TFS / controller

hackfest/tfs-descriptors/device-all.json

+4 −4

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -26,7 +26,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -43,7 +43,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -60,7 +60,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},

hackfest/tfs-descriptors/device-netconf-openconfig.json

+1 −1

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 15}
		}}}
		]},

hackfest/tfs-descriptors/old/device.json

+3 −3

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8301"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],
		@@ -39,7 +39,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8302"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],
		@@ -51,7 +51,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8303"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],

src/device/service/drivers/openconfig/OpenConfigDriver.py

+54 −53

Original line number	Diff line number	Diff line
		@@ -38,8 +38,6 @@ logging.getLogger('apscheduler.executors.default').setLevel(logging.INFO if DEBU
		logging.getLogger('apscheduler.scheduler').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)
		logging.getLogger('monitoring-client').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

		LOGGER = logging.getLogger(__name__)

		RE_GET_ENDPOINT_FROM_INTERFACE_KEY = re.compile(r'.interface\[([^\]]+)\].')
		RE_GET_ENDPOINT_FROM_INTERFACE_XPATH = re.compile(r".interface\[oci\:name\='([^\]]+)'\].")

		@@ -68,7 +66,7 @@ class NetconfSessionHandler:
		self.__look_for_keys = settings.get('look_for_keys', True)
		self.__allow_agent = settings.get('allow_agent', True)
		self.__force_running = settings.get('force_running', False)
		self.__commit_per_delete = settings.get('delete_rule', False)
		self.__commit_per_rule = settings.get('commit_per_rule', False)
		self.__device_params = settings.get('device_params', {})
		self.__manager_params = settings.get('manager_params', {})
		self.__nc_params = settings.get('nc_params', {})
		@@ -94,7 +92,7 @@ class NetconfSessionHandler:
		def use_candidate(self): return self.__candidate_supported and not self.__force_running

		@property
		def commit_per_rule(self): return self.__commit_per_delete
		def commit_per_rule(self): return self.__commit_per_rule

		@property
		def vendor(self): return self.__vendor
		@@ -141,8 +139,9 @@ def compute_delta_sample(previous_sample, previous_timestamp, current_sample, cu
		return delta_sample

		class SamplesCache:
		def __init__(self, netconf_handler : NetconfSessionHandler) -> None:
		def __init__(self, netconf_handler : NetconfSessionHandler, logger : logging.Logger) -> None:
		self.__netconf_handler = netconf_handler
		self.__logger = logger
		self.__lock = threading.Lock()
		self.__timestamp = None
		self.__absolute_samples = {}
		@@ -166,7 +165,7 @@ class SamplesCache:
		self.__absolute_samples[interface] = samples
		self.__timestamp = now
		except: # pylint: disable=bare-except
		LOGGER.exception('Error collecting samples')
		self.__logger.exception('Error collecting samples')

		def get(self, resource_key : str) -> Tuple[float, Dict]:
		self._refresh_samples()
		@@ -176,31 +175,33 @@ class SamplesCache:
		interface = match.group(1)
		return self.__timestamp, copy.deepcopy(self.__delta_samples.get(interface, {}))

		def do_sampling(samples_cache : SamplesCache, resource_key : str, out_samples : queue.Queue) -> None:
		def do_sampling(
		samples_cache : SamplesCache, logger : logging.Logger, resource_key : str, out_samples : queue.Queue
		) -> None:
		try:
		timestamp, samples = samples_cache.get(resource_key)
		counter_name = resource_key.split('/')[-1].split(':')[-1]
		value = samples.get(counter_name)
		if value is None:
		LOGGER.warning('[do_sampling] value not found for {:s}'.format(resource_key))
		logger.warning('[do_sampling] value not found for {:s}'.format(resource_key))
		return
		sample = (timestamp, resource_key, value)
		out_samples.put_nowait(sample)
		except: # pylint: disable=bare-except
		LOGGER.exception('Error retrieving samples')
		logger.exception('Error retrieving samples')

		def edit_config(
		netconf_handler : NetconfSessionHandler, resources : List[Tuple[str, Any]], delete=False, commit_per_rule= False,
		target='running', default_operation='merge', test_option=None, error_option=None,
		netconf_handler : NetconfSessionHandler, logger : logging.Logger, resources : List[Tuple[str, Any]], delete=False,
		commit_per_rule=False, target='running', default_operation='merge', test_option=None, error_option=None,
		format='xml' # pylint: disable=redefined-builtin
		):
		str_method = 'DeleteConfig' if delete else 'SetConfig'
		LOGGER.info('[{:s}] resources = {:s}'.format(str_method, str(resources)))
		#logger.debug('[{:s}] resources = {:s}'.format(str_method, str(resources)))
		results = [None for _ in resources]
		for i,resource in enumerate(resources):
		str_resource_name = 'resources[#{:d}]'.format(i)
		try:
		LOGGER.info('[{:s}] resource = {:s}'.format(str_method, str(resource)))
		#logger.debug('[{:s}] resource = {:s}'.format(str_method, str(resource)))
		chk_type(str_resource_name, resource, (list, tuple))
		chk_length(str_resource_name, resource, min_length=2, max_length=2)
		resource_key,resource_value = resource
		@@ -208,8 +209,8 @@ def edit_config(
		str_config_message = compose_config(
		resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor)
		if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)
		LOGGER.info('[{:s}] str_config_message[{:d}] = {:s}'.format(
		str_method, len(str_config_message), str(str_config_message)))
		#logger.debug('[{:s}] str_config_message[{:d}] = {:s}'.format(
		# str_method, len(str_config_message), str(str_config_message)))
		netconf_handler.edit_config(
		config=str_config_message, target=target, default_operation=default_operation,
		test_option=test_option, error_option=error_option, format=format)
		@@ -219,8 +220,16 @@ def edit_config(
		except Exception as e: # pylint: disable=broad-except
		str_operation = 'preparing' if target == 'candidate' else ('deleting' if delete else 'setting')
		msg = '[{:s}] Exception {:s} {:s}: {:s}'
		LOGGER.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))
		logger.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))
		results[i] = e # if validation fails, store the exception

		if not commit_per_rule:
		try:
		netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		msg = '[{:s}] Exception committing: {:s}'
		logger.exception(msg.format(str_method, str_operation, str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		return results

		HISTOGRAM_BUCKETS = (
		@@ -243,6 +252,7 @@ METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION

		class OpenConfigDriver(_Driver):
		def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called
		self.__logger = logging.getLogger('{:s}:[{:s}:{:s}]'.format(str(__name__), str(address), str(port)))
		self.__lock = threading.Lock()
		#self.__initial = TreeNode('.')
		#self.__running = TreeNode('.')
		@@ -257,7 +267,7 @@ class OpenConfigDriver(_Driver):
		timezone=pytz.utc)
		self.__out_samples = queue.Queue()
		self.__netconf_handler : NetconfSessionHandler = NetconfSessionHandler(address, port, **settings)
		self.__samples_cache = SamplesCache(self.__netconf_handler)
		self.__samples_cache = SamplesCache(self.__netconf_handler, self.__logger)

		def Connect(self) -> bool:
		with self.__lock:
		@@ -295,13 +305,14 @@ class OpenConfigDriver(_Driver):
		try:
		chk_string(str_resource_name, resource_key, allow_empty=False)
		str_filter = get_filter(resource_key)
		LOGGER.info('[GetConfig] str_filter = {:s}'.format(str(str_filter)))
		#self.__logger.debug('[GetConfig] str_filter = {:s}'.format(str(str_filter)))
		if str_filter is None: str_filter = resource_key
		xml_data = self.__netconf_handler.get(filter=str_filter).data_ele
		if isinstance(xml_data, Exception): raise xml_data
		results.extend(parse(resource_key, xml_data))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception retrieving {:s}: {:s}'.format(str_resource_name, str(resource_key)))
		MSG = 'Exception retrieving {:s}: {:s}'
		self.__logger.exception(MSG.format(str_resource_name, str(resource_key)))
		results.append((resource_key, e)) # if validation fails, store the exception
		return results

		@@ -312,17 +323,11 @@ class OpenConfigDriver(_Driver):
		with self.__lock:
		if self.__netconf_handler.use_candidate:
		with self.__netconf_handler.locked(target='candidate'):
		if self.__netconf_handler.commit_per_rule:
		results = edit_config(self.__netconf_handler, resources, target='candidate', commit_per_rule= True)
		else:
		results = edit_config(self.__netconf_handler, resources, target='candidate')
		try:
		self.__netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('[SetConfig] Exception commiting resources: {:s}'.format(str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		results = edit_config(
		self.__netconf_handler, self.__logger, resources, target='candidate',
		commit_per_rule=self.__netconf_handler.commit_per_rule)
		else:
		results = edit_config(self.__netconf_handler, resources)
		results = edit_config(self.__netconf_handler, self.__logger, resources)
		return results

		@metered_subclass_method(METRICS_POOL)
		@@ -332,17 +337,11 @@ class OpenConfigDriver(_Driver):
		with self.__lock:
		if self.__netconf_handler.use_candidate:
		with self.__netconf_handler.locked(target='candidate'):
		if self.__netconf_handler.commit_per_rule:
		results = edit_config(self.__netconf_handler, resources, target='candidate', delete=True, commit_per_rule= True)
		else:
		results = edit_config(self.__netconf_handler, resources, target='candidate', delete=True)
		try:
		self.__netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('[DeleteConfig] Exception commiting resources: {:s}'.format(str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		results = edit_config(
		self.__netconf_handler, self.__logger, resources, target='candidate', delete=True,
		commit_per_rule=self.__netconf_handler.commit_per_rule)
		else:
		results = edit_config(self.__netconf_handler, resources, delete=True)
		results = edit_config(self.__netconf_handler, self.__logger, resources, delete=True)
		return results

		@metered_subclass_method(METRICS_POOL)
		@@ -363,7 +362,8 @@ class OpenConfigDriver(_Driver):
		chk_float(str_subscription_name + '.sampling_duration', sampling_duration, min_value=0)
		chk_float(str_subscription_name + '.sampling_interval', sampling_interval, min_value=0)
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception validating {:s}: {:s}'.format(str_subscription_name, str(resource_key)))
		MSG = 'Exception validating {:s}: {:s}'
		self.__logger.exception(MSG.format(str_subscription_name, str(resource_key)))
		results.append(e) # if validation fails, store the exception
		continue

		@@ -374,7 +374,7 @@ class OpenConfigDriver(_Driver):

		job_id = 'k={:s}/d={:f}/i={:f}'.format(resource_key, sampling_duration, sampling_interval)
		job = self.__scheduler.add_job(
		do_sampling, args=(self.__samples_cache, resource_key, self.__out_samples),
		do_sampling, args=(self.__samples_cache, self.__logger, resource_key, self.__out_samples),
		kwargs={}, id=job_id, trigger='interval', seconds=sampling_interval,
		start_date=start_date, end_date=end_date, timezone=pytz.utc)

		@@ -401,7 +401,8 @@ class OpenConfigDriver(_Driver):
		chk_float(str_subscription_name + '.sampling_duration', sampling_duration, min_value=0)
		chk_float(str_subscription_name + '.sampling_interval', sampling_interval, min_value=0)
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception validating {:s}: {:s}'.format(str_subscription_name, str(resource_key)))
		MSG = 'Exception validating {:s}: {:s}'
		self.__logger.exception(MSG.format(str_subscription_name, str(resource_key)))
		results.append(e) # if validation fails, store the exception
		continue

src/device/service/drivers/openconfig/templates/interface/edit_config.xml

+1 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@
		{% if operation is defined and operation != 'delete' %}
		<config>
		<name>{{name}}</name>
		<type xmlns:ianaift="urn:ietf:params:xml:ns:yang:iana-if-type">ianaift:{{type}}</type>
		<description></description>
		<mtu>{{mtu}}</mtu>
		</config>