Merge branch 'develop' into feat/compute-component (a25e2a2a) · Commits · TFS / controller

hackfest/tfs-descriptors/device-all.json

+4 −4

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -26,7 +26,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -43,7 +43,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},
		@@ -60,7 +60,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 120}
		}}}
		]},

hackfest/tfs-descriptors/device-netconf-openconfig.json

+1 −1

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {
		"username": "admin", "password": "admin",
		"force_running": false, "hostkey_verify": false, "look_for_keys": false,
		"allow_agent": false, "delete_rule": true, "device_params": {"name": "default"},
		"allow_agent": false, "commit_per_rule": true, "device_params": {"name": "default"},
		"manager_params": {"timeout" : 15}
		}}}
		]},

hackfest/tfs-descriptors/old/device.json

+3 −3

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8301"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],
		@@ -39,7 +39,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8302"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],
		@@ -51,7 +51,7 @@
		"device_config": {"config_rules": [
		{"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.15"}},
		{"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8303"}},
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"delete_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		{"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": "{\"username\": \"admin\", \"password\": \"admin\", \"force_running\": true, \"hostkey_verify\": false, \"look_for_keys\": false, \"allow_agent\": true, \"commit_per_rule\": false, \"device_params\" : {\"name\": \"default\"}, \"manager_params\" : {\"timeout\": 15}}"}}
		]},
		"device_operational_status": 1,
		"device_drivers": [1],

src/common/method_wrappers/tests/deploy_specs.sh

+69 −13

Original line number	Diff line number	Diff line
		#!/bin/bash
		# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		@@ -12,21 +13,19 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		# Set the URL of your local Docker registry where the images will be uploaded to.
		export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/"

		# ----- TeraFlowSDN ------------------------------------------------------------

		# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to.
		export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/"

		# Set the list of components, separated by spaces, you want to build images for, and deploy.
		# Supported components are:
		# context device automation policy service compute monitoring webui
		# interdomain slice pathcomp dlt
		# dbscanserving opticalattackmitigator opticalattackdetector
		# l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector
		export TFS_COMPONENTS="context device pathcomp service slice webui load_generator" # automation monitoring compute dlt
		export TFS_COMPONENTS="context device pathcomp service slice webui load_generator"

		# Set the tag you want to use for your images.
		export TFS_IMAGE_TAG="dev"

		# Set the name of the Kubernetes namespace to deploy to.
		# Set the name of the Kubernetes namespace to deploy TFS to.
		export TFS_K8S_NAMESPACE="tfs"

		# Set additional manifest files to be applied after the deployment
		@@ -35,6 +34,63 @@ export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicem
		# Set the new Grafana admin password
		export TFS_GRAFANA_PASSWORD="admin123+"

		# If not already set, disable skip-build flag.
		# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
		export TFS_SKIP_BUILD="NO" #${TFS_SKIP_BUILD:-"YES"}
		# Disable skip-build flag to rebuild the Docker images.
		export TFS_SKIP_BUILD=""


		# ----- CockroachDB ------------------------------------------------------------

		# Set the namespace where CockroackDB will be deployed.
		export CRDB_NAMESPACE="crdb"

		# Set the database username to be used by Context.
		export CRDB_USERNAME="tfs"

		# Set the database user's password to be used by Context.
		export CRDB_PASSWORD="tfs123"

		# Set the database name to be used by Context.
		export CRDB_DATABASE="tfs"

		# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing.
		# See ./deploy/all.sh or ./deploy/crdb.sh for additional details
		export CRDB_DEPLOY_MODE="single"

		# Disable flag for dropping database, if it exists.
		export CRDB_DROP_DATABASE_IF_EXISTS=""

		# Disable flag for re-deploying CockroachDB from scratch.
		export CRDB_REDEPLOY=""


		# ----- NATS -------------------------------------------------------------------

		# Set the namespace where NATS will be deployed.
		export NATS_NAMESPACE="nats"

		# Disable flag for re-deploying NATS from scratch.
		export NATS_REDEPLOY=""


		# ----- QuestDB ----------------------------------------------------------------

		# Set the namespace where QuestDB will be deployed.
		export QDB_NAMESPACE="qdb"

		# Set the database username to be used for QuestDB.
		export QDB_USERNAME="admin"

		# Set the database user's password to be used for QuestDB.
		export QDB_PASSWORD="quest"

		# Set the table name to be used by Monitoring for KPIs.
		export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis"

		# Set the table name to be used by Slice for plotting groups.
		export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups"

		# Disable flag for dropping tables if they exist.
		export QDB_DROP_TABLES_IF_EXIST=""

		# Disable flag for re-deploying QuestDB from scratch.
		export QDB_REDEPLOY=""

src/device/service/drivers/openconfig/OpenConfigDriver.py

+54 −52

Original line number	Diff line number	Diff line
		@@ -38,8 +38,6 @@ logging.getLogger('apscheduler.executors.default').setLevel(logging.INFO if DEBU
		logging.getLogger('apscheduler.scheduler').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)
		logging.getLogger('monitoring-client').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

		LOGGER = logging.getLogger(__name__)

		RE_GET_ENDPOINT_FROM_INTERFACE_KEY = re.compile(r'.interface\[([^\]]+)\].')
		RE_GET_ENDPOINT_FROM_INTERFACE_XPATH = re.compile(r".interface\[oci\:name\='([^\]]+)'\].")

		@@ -68,7 +66,7 @@ class NetconfSessionHandler:
		self.__look_for_keys = settings.get('look_for_keys', True)
		self.__allow_agent = settings.get('allow_agent', True)
		self.__force_running = settings.get('force_running', False)
		self.__commit_per_delete = settings.get('delete_rule', False)
		self.__commit_per_rule = settings.get('commit_per_rule', False)
		self.__device_params = settings.get('device_params', {})
		self.__manager_params = settings.get('manager_params', {})
		self.__nc_params = settings.get('nc_params', {})
		@@ -94,7 +92,7 @@ class NetconfSessionHandler:
		def use_candidate(self): return self.__candidate_supported and not self.__force_running

		@property
		def commit_per_rule(self): return self.__commit_per_delete
		def commit_per_rule(self): return self.__commit_per_rule

		@property
		def vendor(self): return self.__vendor
		@@ -141,8 +139,9 @@ def compute_delta_sample(previous_sample, previous_timestamp, current_sample, cu
		return delta_sample

		class SamplesCache:
		def __init__(self, netconf_handler : NetconfSessionHandler) -> None:
		def __init__(self, netconf_handler : NetconfSessionHandler, logger : logging.Logger) -> None:
		self.__netconf_handler = netconf_handler
		self.__logger = logger
		self.__lock = threading.Lock()
		self.__timestamp = None
		self.__absolute_samples = {}
		@@ -166,7 +165,7 @@ class SamplesCache:
		self.__absolute_samples[interface] = samples
		self.__timestamp = now
		except: # pylint: disable=bare-except
		LOGGER.exception('Error collecting samples')
		self.__logger.exception('Error collecting samples')

		def get(self, resource_key : str) -> Tuple[float, Dict]:
		self._refresh_samples()
		@@ -176,31 +175,33 @@ class SamplesCache:
		interface = match.group(1)
		return self.__timestamp, copy.deepcopy(self.__delta_samples.get(interface, {}))

		def do_sampling(samples_cache : SamplesCache, resource_key : str, out_samples : queue.Queue) -> None:
		def do_sampling(
		samples_cache : SamplesCache, logger : logging.Logger, resource_key : str, out_samples : queue.Queue
		) -> None:
		try:
		timestamp, samples = samples_cache.get(resource_key)
		counter_name = resource_key.split('/')[-1].split(':')[-1]
		value = samples.get(counter_name)
		if value is None:
		LOGGER.warning('[do_sampling] value not found for {:s}'.format(resource_key))
		logger.warning('[do_sampling] value not found for {:s}'.format(resource_key))
		return
		sample = (timestamp, resource_key, value)
		out_samples.put_nowait(sample)
		except: # pylint: disable=bare-except
		LOGGER.exception('Error retrieving samples')
		logger.exception('Error retrieving samples')

		def edit_config(
		netconf_handler : NetconfSessionHandler, resources : List[Tuple[str, Any]], delete=False, commit_per_rule= False,
		target='running', default_operation='merge', test_option=None, error_option=None,
		netconf_handler : NetconfSessionHandler, logger : logging.Logger, resources : List[Tuple[str, Any]], delete=False,
		commit_per_rule=False, target='running', default_operation='merge', test_option=None, error_option=None,
		format='xml' # pylint: disable=redefined-builtin
		):
		str_method = 'DeleteConfig' if delete else 'SetConfig'
		LOGGER.info('[{:s}] resources = {:s}'.format(str_method, str(resources)))
		#logger.debug('[{:s}] resources = {:s}'.format(str_method, str(resources)))
		results = [None for _ in resources]
		for i,resource in enumerate(resources):
		str_resource_name = 'resources[#{:d}]'.format(i)
		try:
		LOGGER.info('[{:s}] resource = {:s}'.format(str_method, str(resource)))
		logger.debug('[{:s}] resource = {:s}'.format(str_method, str(resource)))
		chk_type(str_resource_name, resource, (list, tuple))
		chk_length(str_resource_name, resource, min_length=2, max_length=2)
		resource_key,resource_value = resource
		@@ -208,7 +209,7 @@ def edit_config(
		str_config_message = compose_config(
		resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor)
		if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)
		LOGGER.info('[{:s}] str_config_message[{:d}] = {:s}'.format(
		logger.debug('[{:s}] str_config_message[{:d}] = {:s}'.format(
		str_method, len(str_config_message), str(str_config_message)))
		netconf_handler.edit_config(
		config=str_config_message, target=target, default_operation=default_operation,
		@@ -219,8 +220,17 @@ def edit_config(
		except Exception as e: # pylint: disable=broad-except
		str_operation = 'preparing' if target == 'candidate' else ('deleting' if delete else 'setting')
		msg = '[{:s}] Exception {:s} {:s}: {:s}'
		LOGGER.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))
		logger.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))
		results[i] = e # if validation fails, store the exception

		if not commit_per_rule:
		try:
		netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		msg = '[{:s}] Exception committing: {:s}'
		str_operation = 'preparing' if target == 'candidate' else ('deleting' if delete else 'setting')
		logger.exception(msg.format(str_method, str_operation, str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		return results

		HISTOGRAM_BUCKETS = (
		@@ -243,6 +253,7 @@ METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION

		class OpenConfigDriver(_Driver):
		def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called
		self.__logger = logging.getLogger('{:s}:[{:s}:{:s}]'.format(str(__name__), str(address), str(port)))
		self.__lock = threading.Lock()
		#self.__initial = TreeNode('.')
		#self.__running = TreeNode('.')
		@@ -257,7 +268,7 @@ class OpenConfigDriver(_Driver):
		timezone=pytz.utc)
		self.__out_samples = queue.Queue()
		self.__netconf_handler : NetconfSessionHandler = NetconfSessionHandler(address, port, **settings)
		self.__samples_cache = SamplesCache(self.__netconf_handler)
		self.__samples_cache = SamplesCache(self.__netconf_handler, self.__logger)

		def Connect(self) -> bool:
		with self.__lock:
		@@ -295,13 +306,14 @@ class OpenConfigDriver(_Driver):
		try:
		chk_string(str_resource_name, resource_key, allow_empty=False)
		str_filter = get_filter(resource_key)
		LOGGER.info('[GetConfig] str_filter = {:s}'.format(str(str_filter)))
		#self.__logger.debug('[GetConfig] str_filter = {:s}'.format(str(str_filter)))
		if str_filter is None: str_filter = resource_key
		xml_data = self.__netconf_handler.get(filter=str_filter).data_ele
		if isinstance(xml_data, Exception): raise xml_data
		results.extend(parse(resource_key, xml_data))
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception retrieving {:s}: {:s}'.format(str_resource_name, str(resource_key)))
		MSG = 'Exception retrieving {:s}: {:s}'
		self.__logger.exception(MSG.format(str_resource_name, str(resource_key)))
		results.append((resource_key, e)) # if validation fails, store the exception
		return results

		@@ -312,17 +324,11 @@ class OpenConfigDriver(_Driver):
		with self.__lock:
		if self.__netconf_handler.use_candidate:
		with self.__netconf_handler.locked(target='candidate'):
		if self.__netconf_handler.commit_per_rule:
		results = edit_config(self.__netconf_handler, resources, target='candidate', commit_per_rule= True)
		results = edit_config(
		self.__netconf_handler, self.__logger, resources, target='candidate',
		commit_per_rule=self.__netconf_handler.commit_per_rule)
		else:
		results = edit_config(self.__netconf_handler, resources, target='candidate')
		try:
		self.__netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('[SetConfig] Exception commiting resources: {:s}'.format(str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		else:
		results = edit_config(self.__netconf_handler, resources)
		results = edit_config(self.__netconf_handler, self.__logger, resources)
		return results

		@metered_subclass_method(METRICS_POOL)
		@@ -332,17 +338,11 @@ class OpenConfigDriver(_Driver):
		with self.__lock:
		if self.__netconf_handler.use_candidate:
		with self.__netconf_handler.locked(target='candidate'):
		if self.__netconf_handler.commit_per_rule:
		results = edit_config(self.__netconf_handler, resources, target='candidate', delete=True, commit_per_rule= True)
		else:
		results = edit_config(self.__netconf_handler, resources, target='candidate', delete=True)
		try:
		self.__netconf_handler.commit()
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('[DeleteConfig] Exception commiting resources: {:s}'.format(str(resources)))
		results = [e for _ in resources] # if commit fails, set exception in each resource
		results = edit_config(
		self.__netconf_handler, self.__logger, resources, target='candidate', delete=True,
		commit_per_rule=self.__netconf_handler.commit_per_rule)
		else:
		results = edit_config(self.__netconf_handler, resources, delete=True)
		results = edit_config(self.__netconf_handler, self.__logger, resources, delete=True)
		return results

		@metered_subclass_method(METRICS_POOL)
		@@ -363,7 +363,8 @@ class OpenConfigDriver(_Driver):
		chk_float(str_subscription_name + '.sampling_duration', sampling_duration, min_value=0)
		chk_float(str_subscription_name + '.sampling_interval', sampling_interval, min_value=0)
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception validating {:s}: {:s}'.format(str_subscription_name, str(resource_key)))
		MSG = 'Exception validating {:s}: {:s}'
		self.__logger.exception(MSG.format(str_subscription_name, str(resource_key)))
		results.append(e) # if validation fails, store the exception
		continue

		@@ -374,7 +375,7 @@ class OpenConfigDriver(_Driver):

		job_id = 'k={:s}/d={:f}/i={:f}'.format(resource_key, sampling_duration, sampling_interval)
		job = self.__scheduler.add_job(
		do_sampling, args=(self.__samples_cache, resource_key, self.__out_samples),
		do_sampling, args=(self.__samples_cache, self.__logger, resource_key, self.__out_samples),
		kwargs={}, id=job_id, trigger='interval', seconds=sampling_interval,
		start_date=start_date, end_date=end_date, timezone=pytz.utc)

		@@ -401,7 +402,8 @@ class OpenConfigDriver(_Driver):
		chk_float(str_subscription_name + '.sampling_duration', sampling_duration, min_value=0)
		chk_float(str_subscription_name + '.sampling_interval', sampling_interval, min_value=0)
		except Exception as e: # pylint: disable=broad-except
		LOGGER.exception('Exception validating {:s}: {:s}'.format(str_subscription_name, str(resource_key)))
		MSG = 'Exception validating {:s}: {:s}'
		self.__logger.exception(MSG.format(str_subscription_name, str(resource_key)))
		results.append(e) # if validation fails, store the exception
		continue