diff --git a/scripts/run_tests_locally-device.sh b/scripts/run_tests_locally-device.sh new file mode 100755 index 0000000000000000000000000000000000000000..ba6c0b6a58031720addc17cc0de9169e592099f5 --- /dev/null +++ b/scripts/run_tests_locally-device.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc + +# Run unitary tests and analyze coverage of code at same time + +# Useful flags for pytest: +#-o log_cli=true -o log_file=device.log -o log_file_level=DEBUG + +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + device/tests/test_unitary.py diff --git a/src/device/service/driver_api/_Driver.py b/src/device/service/driver_api/_Driver.py index 83462bb3946cdc0dd8ec4af68c3bac35f94f0cc3..f30165a178a5946c414157da5d09df07bf060a39 100644 --- a/src/device/service/driver_api/_Driver.py +++ b/src/device/service/driver_api/_Driver.py @@ -20,6 +20,7 @@ from typing import Any, Iterator, List, Optional, Tuple, Union RESOURCE_ENDPOINTS = '__endpoints__' RESOURCE_INTERFACES = '__interfaces__' RESOURCE_NETWORK_INSTANCES = '__network_instances__' +RESOURCE_ROUTING_POLICIES = '__routing_policies__' class _Driver: def __init__(self, address : str, port : int, **settings) -> None: diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index e89c865037694f267280fb13db02e46f91c5ff56..7f582c4880bafd08aee0204c7498ea3a3e7ad279 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -13,22 +13,23 @@ # limitations under the License. import anytree, copy, logging, pytz, queue, re, threading -import lxml.etree as ET +#import lxml.etree as ET from datetime import datetime, timedelta from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.job import Job from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.background import BackgroundScheduler -from netconf_client.connect import connect_ssh +from netconf_client.connect import connect_ssh, Session from netconf_client.ncclient import Manager +from common.tools.client.RetryDecorator import delay_exponential from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float from device.service.driver_api.Exceptions import UnsupportedResourceKeyException from device.service.driver_api._Driver import _Driver -from device.service.driver_api.AnyTreeTools import TreeNode, dump_subtree, get_subnode, set_subnode_value -from device.service.drivers.openconfig.Tools import xml_pretty_print, xml_to_dict, xml_to_file -from device.service.drivers.openconfig.templates import ( - ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filter, parse) +from device.service.driver_api.AnyTreeTools import TreeNode, get_subnode, set_subnode_value #dump_subtree +#from .Tools import xml_pretty_print, xml_to_dict, xml_to_file +from .templates import ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filter, parse +from .RetryDecorator import retry DEBUG_MODE = False #logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING) @@ -48,6 +49,51 @@ RE_GET_ENDPOINT_FROM_INTERFACE_XPATH = re.compile(r".*interface\[oci\:name\='([^ SAMPLE_EVICTION_SECONDS = 30.0 # seconds SAMPLE_RESOURCE_KEY = 'interfaces/interface/state/counters' +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class NetconfSessionHandler: + def __init__(self, address : str, port : int, **settings) -> None: + self.__lock = threading.RLock() + self.__connected = threading.Event() + self.__address = address + self.__port = int(port) + self.__username = settings.get('username') + self.__password = settings.get('password') + self.__timeout = int(settings.get('timeout', 120)) + self.__netconf_session : Session = None + self.__netconf_manager : Manager = None + + def connect(self): + with self.__lock: + self.__netconf_session = connect_ssh( + host=self.__address, port=self.__port, username=self.__username, password=self.__password) + self.__netconf_manager = Manager(self.__netconf_session, timeout=self.__timeout) + self.__netconf_manager.set_logger_level(logging.DEBUG if DEBUG_MODE else logging.WARNING) + self.__connected.set() + + def disconnect(self): + if not self.__connected.is_set(): return + with self.__lock: + self.__netconf_manager.close_session() + + @RETRY_DECORATOR + def get(self, filter=None, with_defaults=None): # pylint: disable=redefined-builtin + with self.__lock: + return self.__netconf_manager.get(filter=filter, with_defaults=with_defaults) + + @RETRY_DECORATOR + def edit_config( + self, config, target='running', default_operation=None, test_option=None, + error_option=None, format='xml' # pylint: disable=redefined-builtin + ): + if config == EMPTY_CONFIG: return + with self.__lock: + self.__netconf_manager.edit_config( + config, target=target, default_operation=default_operation, test_option=test_option, + error_option=error_option, format=format) + def compute_delta_sample(previous_sample, previous_timestamp, current_sample, current_timestamp): if previous_sample is None: return None if previous_timestamp is None: return None @@ -68,19 +114,20 @@ def compute_delta_sample(previous_sample, previous_timestamp, current_sample, cu return delta_sample class SamplesCache: - def __init__(self) -> None: + def __init__(self, netconf_handler : NetconfSessionHandler) -> None: + self.__netconf_handler = netconf_handler self.__lock = threading.Lock() self.__timestamp = None self.__absolute_samples = {} self.__delta_samples = {} - def _refresh_samples(self, netconf_manager : Manager) -> None: + def _refresh_samples(self) -> None: with self.__lock: try: now = datetime.timestamp(datetime.utcnow()) if self.__timestamp is not None and (now - self.__timestamp) < SAMPLE_EVICTION_SECONDS: return str_filter = get_filter(SAMPLE_RESOURCE_KEY) - xml_data = netconf_manager.get(filter=str_filter).data_ele + xml_data = self.__netconf_handler.get(filter=str_filter).data_ele interface_samples = parse(SAMPLE_RESOURCE_KEY, xml_data) for interface,samples in interface_samples: match = RE_GET_ENDPOINT_FROM_INTERFACE_KEY.match(interface) @@ -94,19 +141,17 @@ class SamplesCache: except: # pylint: disable=bare-except LOGGER.exception('Error collecting samples') - def get(self, resource_key : str, netconf_manager : Manager) -> Tuple[float, Dict]: - self._refresh_samples(netconf_manager) + def get(self, resource_key : str) -> Tuple[float, Dict]: + self._refresh_samples() match = RE_GET_ENDPOINT_FROM_INTERFACE_XPATH.match(resource_key) with self.__lock: if match is None: return self.__timestamp, {} interface = match.group(1) return self.__timestamp, copy.deepcopy(self.__delta_samples.get(interface, {})) -def do_sampling( - netconf_manager : Manager, samples_cache : SamplesCache, resource_key : str, out_samples : queue.Queue -) -> None: +def do_sampling(samples_cache : SamplesCache, resource_key : str, out_samples : queue.Queue) -> None: try: - timestamp, samples = samples_cache.get(resource_key, netconf_manager) + timestamp, samples = samples_cache.get(resource_key) counter_name = resource_key.split('/')[-1].split(':')[-1] value = samples.get(counter_name) if value is None: @@ -117,19 +162,14 @@ def do_sampling( except: # pylint: disable=bare-except LOGGER.exception('Error retrieving samples') - class OpenConfigDriver(_Driver): def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called - self.__address = address - self.__port = int(port) - self.__settings = settings self.__lock = threading.Lock() #self.__initial = TreeNode('.') #self.__running = TreeNode('.') self.__subscriptions = TreeNode('.') self.__started = threading.Event() self.__terminate = threading.Event() - self.__netconf_manager : Manager = None self.__scheduler = BackgroundScheduler(daemon=True) # scheduler used to emulate sampling events self.__scheduler.configure( jobstores = {'default': MemoryJobStore()}, @@ -137,18 +177,13 @@ class OpenConfigDriver(_Driver): job_defaults = {'coalesce': False, 'max_instances': 3}, timezone=pytz.utc) self.__out_samples = queue.Queue() - self.__samples_cache = SamplesCache() + self.__netconf_handler : NetconfSessionHandler = NetconfSessionHandler(address, port, **settings) + self.__samples_cache = SamplesCache(self.__netconf_handler) def Connect(self) -> bool: with self.__lock: if self.__started.is_set(): return True - username = self.__settings.get('username') - password = self.__settings.get('password') - timeout = int(self.__settings.get('timeout', 120)) - session = connect_ssh( - host=self.__address, port=self.__port, username=username, password=password) - self.__netconf_manager = Manager(session, timeout=timeout) - self.__netconf_manager.set_logger_level(logging.DEBUG if DEBUG_MODE else logging.WARNING) + self.__netconf_handler.connect() # Connect triggers activation of sampling events that will be scheduled based on subscriptions self.__scheduler.start() self.__started.set() @@ -162,7 +197,7 @@ class OpenConfigDriver(_Driver): if not self.__started.is_set(): return True # Disconnect triggers deactivation of sampling events self.__scheduler.shutdown() - self.__netconf_manager.close_session() + self.__netconf_handler.disconnect() return True def GetInitialConfig(self) -> List[Tuple[str, Any]]: @@ -179,8 +214,9 @@ class OpenConfigDriver(_Driver): try: chk_string(str_resource_name, resource_key, allow_empty=False) str_filter = get_filter(resource_key) + LOGGER.info('[GetConfig] str_filter = {:s}'.format(str(str_filter))) if str_filter is None: str_filter = resource_key - xml_data = self.__netconf_manager.get(filter=str_filter).data_ele + xml_data = self.__netconf_handler.get(filter=str_filter).data_ele if isinstance(xml_data, Exception): raise xml_data results.extend(parse(resource_key, xml_data)) except Exception as e: # pylint: disable=broad-except @@ -206,8 +242,7 @@ class OpenConfigDriver(_Driver): if str_config_message is None: raise UnsupportedResourceKeyException(resource_key) LOGGER.info('[SetConfig] str_config_message[{:d}] = {:s}'.format( len(str_config_message), str(str_config_message))) - if str_config_message != EMPTY_CONFIG: - self.__netconf_manager.edit_config(str_config_message, target='running') + self.__netconf_handler.edit_config(str_config_message, target='running') results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception setting {:s}: {:s}'.format(str_resource_name, str(resource))) @@ -232,9 +267,7 @@ class OpenConfigDriver(_Driver): if str_config_message is None: raise UnsupportedResourceKeyException(resource_key) LOGGER.info('[DeleteConfig] str_config_message[{:d}] = {:s}'.format( len(str_config_message), str(str_config_message))) - if str_config_message != EMPTY_CONFIG: - self.__netconf_manager.edit_config(str_config_message, target='running') - self.__netconf_manager.edit_config(str_config_message, target='running') + self.__netconf_handler.edit_config(str_config_message, target='running') results.append(True) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception deleting {:s}: {:s}'.format(str_resource_name, str(resource_key))) @@ -269,7 +302,7 @@ class OpenConfigDriver(_Driver): job_id = 'k={:s}/d={:f}/i={:f}'.format(resource_key, sampling_duration, sampling_interval) job = self.__scheduler.add_job( - do_sampling, args=(self.__netconf_manager, self.__samples_cache, resource_key, self.__out_samples), + do_sampling, args=(self.__samples_cache, resource_key, self.__out_samples), kwargs={}, id=job_id, trigger='interval', seconds=sampling_interval, start_date=start_date, end_date=end_date, timezone=pytz.utc) diff --git a/src/device/service/drivers/openconfig/RetryDecorator.py b/src/device/service/drivers/openconfig/RetryDecorator.py new file mode 100644 index 0000000000000000000000000000000000000000..d8ccddb4d09dd8863cedc2893fb3c6ec4e0491cd --- /dev/null +++ b/src/device/service/drivers/openconfig/RetryDecorator.py @@ -0,0 +1,46 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, time +from common.tools.client.RetryDecorator import delay_linear + +LOGGER = logging.getLogger(__name__) + +def retry(max_retries=0, delay_function=delay_linear(initial=0, increment=0), + prepare_method_name=None, prepare_method_args=[], prepare_method_kwargs={}): + def _reconnect(func): + def wrapper(self, *args, **kwargs): + if prepare_method_name is not None: + prepare_method = getattr(self, prepare_method_name, None) + if prepare_method is None: raise Exception('Prepare Method ({}) not found'.format(prepare_method_name)) + num_try, given_up = 0, False + while not given_up: + try: + return func(self, *args, **kwargs) + except OSError as e: + if str(e) != 'Socket is closed': raise + + num_try += 1 + given_up = num_try > max_retries + if given_up: raise Exception('Giving up... {:d} tries failed'.format(max_retries)) from e + if delay_function is not None: + delay = delay_function(num_try) + time.sleep(delay) + LOGGER.info('Retry {:d}/{:d} after {:f} seconds...'.format(num_try, max_retries, delay)) + else: + LOGGER.info('Retry {:d}/{:d} immediate...'.format(num_try, max_retries)) + + if prepare_method_name is not None: prepare_method(*prepare_method_args, **prepare_method_kwargs) + return wrapper + return _reconnect diff --git a/src/device/service/drivers/openconfig/templates/EndPoints.py b/src/device/service/drivers/openconfig/templates/EndPoints.py index 192d0b3de080c363075959a3f50063a4e0732eaa..c11b1669d5b4cf3ca47986817ded28f75ae8358f 100644 --- a/src/device/service/drivers/openconfig/templates/EndPoints.py +++ b/src/device/service/drivers/openconfig/templates/EndPoints.py @@ -47,5 +47,5 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_collection(endpoint, 'sample_types', sample_types) if len(endpoint) == 0: continue - response.append(('endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) + response.append(('/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) return response diff --git a/src/device/service/drivers/openconfig/templates/Interfaces.py b/src/device/service/drivers/openconfig/templates/Interfaces.py index 5044ff16ff895a43effda444ce90f43246a41809..33f977524c6f65655fbe17f6d2d95a7cfc223967 100644 --- a/src/device/service/drivers/openconfig/templates/Interfaces.py +++ b/src/device/service/drivers/openconfig/templates/Interfaces.py @@ -81,11 +81,11 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #add_value_from_collection(subinterface, 'ipv4_addresses', ipv4_addresses) if len(subinterface) == 0: continue - resource_key = 'interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) + resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) response.append((resource_key, subinterface)) if len(interface) == 0: continue - response.append(('interface[{:s}]'.format(interface['name']), interface)) + response.append(('/interface[{:s}]'.format(interface['name']), interface)) return response @@ -124,6 +124,6 @@ def parse_counters(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('[parse_counters] interface = {:s}'.format(str(interface))) if len(interface) == 0: continue - response.append(('interface[{:s}]'.format(interface['name']), interface)) + response.append(('/interface[{:s}]'.format(interface['name']), interface)) return response diff --git a/src/device/service/drivers/openconfig/templates/Namespace.py b/src/device/service/drivers/openconfig/templates/Namespace.py index a557adf79c8a82e427c2903ffc71951f0777d164..35be5827db892541847a3c02af42e2fd08ee0e1d 100644 --- a/src/device/service/drivers/openconfig/templates/Namespace.py +++ b/src/device/service/drivers/openconfig/templates/Namespace.py @@ -15,6 +15,7 @@ NAMESPACE_NETCONF = 'urn:ietf:params:xml:ns:netconf:base:1.0' +NAMESPACE_BGP_POLICY = 'http://openconfig.net/yang/bgp-policy' NAMESPACE_INTERFACES = 'http://openconfig.net/yang/interfaces' NAMESPACE_INTERFACES_IP = 'http://openconfig.net/yang/interfaces/ip' NAMESPACE_NETWORK_INSTANCE = 'http://openconfig.net/yang/network-instance' @@ -22,10 +23,14 @@ NAMESPACE_NETWORK_INSTANCE_TYPES = 'http://openconfig.net/yang/network-instance- NAMESPACE_OPENCONFIG_TYPES = 'http://openconfig.net/yang/openconfig-types' NAMESPACE_PLATFORM = 'http://openconfig.net/yang/platform' NAMESPACE_PLATFORM_PORT = 'http://openconfig.net/yang/platform/port' +NAMESPACE_POLICY_TYPES = 'http://openconfig.net/yang/policy-types' +NAMESPACE_POLICY_TYPES_2 = 'http://openconfig.net/yang/policy_types' +NAMESPACE_ROUTING_POLICY = 'http://openconfig.net/yang/routing-policy' NAMESPACE_VLAN = 'http://openconfig.net/yang/vlan' NAMESPACES = { 'nc' : NAMESPACE_NETCONF, + 'ocbp' : NAMESPACE_BGP_POLICY, 'oci' : NAMESPACE_INTERFACES, 'ociip': NAMESPACE_INTERFACES_IP, 'ocni' : NAMESPACE_NETWORK_INSTANCE, @@ -33,5 +38,8 @@ NAMESPACES = { 'ococt': NAMESPACE_OPENCONFIG_TYPES, 'ocp' : NAMESPACE_PLATFORM, 'ocpp' : NAMESPACE_PLATFORM_PORT, + 'ocpt' : NAMESPACE_POLICY_TYPES, + 'ocpt2': NAMESPACE_POLICY_TYPES_2, + 'ocrp' : NAMESPACE_ROUTING_POLICY, 'ocv' : NAMESPACE_VLAN, } diff --git a/src/device/service/drivers/openconfig/templates/NetworkInstances.py b/src/device/service/drivers/openconfig/templates/NetworkInstances.py index 647647022133ef7bc3d8ed44d1ac3b6fc6bf79d0..b091a0d206195a6c2ce94008628071cd9e30944f 100644 --- a/src/device/service/drivers/openconfig/templates/NetworkInstances.py +++ b/src/device/service/drivers/openconfig/templates/NetworkInstances.py @@ -20,6 +20,12 @@ from .Tools import add_value_from_collection, add_value_from_tag LOGGER = logging.getLogger(__name__) XPATH_NETWORK_INSTANCES = "//ocni:network-instances/ocni:network-instance" +XPATH_NI_PROTOCOLS = ".//ocni:protocols/ocni:protocol" +XPATH_NI_TABLE_CONNECTS = ".//ocni:table-connections/ocni:table-connection" + +XPATH_NI_IIP_AP = ".//ocni:inter-instance-policies/ocni:apply-policy" +XPATH_NI_IIP_AP_IMPORT = ".//ocni:config/ocni:import-policy" +XPATH_NI_IIP_AP_EXPORT = ".//ocni:config/ocni:export-policy" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] @@ -45,5 +51,78 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #add_value_from_collection(network_instance, 'address_families', ni_address_families) if len(network_instance) == 0: continue - response.append(('network_instance[{:s}]'.format(network_instance['name']), network_instance)) + response.append(('/network_instance[{:s}]'.format(network_instance['name']), network_instance)) + + for xml_protocol in xml_network_instance.xpath(XPATH_NI_PROTOCOLS, namespaces=NAMESPACES): + #LOGGER.info('xml_protocol = {:s}'.format(str(ET.tostring(xml_protocol)))) + + protocol = {} + add_value_from_tag(protocol, 'name', ni_name) + + identifier = xml_protocol.find('ocni:identifier', namespaces=NAMESPACES) + if identifier is None: identifier = xml_protocol.find('ocpt:identifier', namespaces=NAMESPACES) + if identifier is None: identifier = xml_protocol.find('ocpt2:identifier', namespaces=NAMESPACES) + if identifier is None or identifier.text is None: continue + add_value_from_tag(protocol, 'identifier', identifier, cast=lambda s: s.replace('oc-pol-types:', '')) + + name = xml_protocol.find('ocni:name', namespaces=NAMESPACES) + add_value_from_tag(protocol, 'protocol_name', name) + + if protocol['identifier'] == 'BGP': + bgp_as = xml_protocol.find('ocni:bgp/ocni:global/ocni:config/ocni:as', namespaces=NAMESPACES) + add_value_from_tag(protocol, 'as', bgp_as, cast=int) + + resource_key = '/network_instance[{:s}]/protocols[{:s}]'.format( + network_instance['name'], protocol['identifier']) + response.append((resource_key, protocol)) + + for xml_table_connection in xml_network_instance.xpath(XPATH_NI_TABLE_CONNECTS, namespaces=NAMESPACES): + #LOGGER.info('xml_table_connection = {:s}'.format(str(ET.tostring(xml_table_connection)))) + + table_connection = {} + add_value_from_tag(table_connection, 'name', ni_name) + + src_protocol = xml_table_connection.find('ocni:src-protocol', namespaces=NAMESPACES) + add_value_from_tag(table_connection, 'src_protocol', src_protocol, + cast=lambda s: s.replace('oc-pol-types:', '')) + + dst_protocol = xml_table_connection.find('ocni:dst-protocol', namespaces=NAMESPACES) + add_value_from_tag(table_connection, 'dst_protocol', dst_protocol, + cast=lambda s: s.replace('oc-pol-types:', '')) + + address_family = xml_table_connection.find('ocni:address-family', namespaces=NAMESPACES) + add_value_from_tag(table_connection, 'address_family', address_family, + cast=lambda s: s.replace('oc-types:', '')) + + default_import_policy = xml_table_connection.find('ocni:default-import-policy', namespaces=NAMESPACES) + add_value_from_tag(table_connection, 'default_import_policy', default_import_policy) + + resource_key = '/network_instance[{:s}]/table_connections[{:s}][{:s}][{:s}]'.format( + network_instance['name'], table_connection['src_protocol'], table_connection['dst_protocol'], + table_connection['address_family']) + response.append((resource_key, table_connection)) + + for xml_iip_ap in xml_network_instance.xpath(XPATH_NI_IIP_AP, namespaces=NAMESPACES): + #LOGGER.info('xml_iip_ap = {:s}'.format(str(ET.tostring(xml_iip_ap)))) + + for xml_import_policy in xml_iip_ap.xpath(XPATH_NI_IIP_AP_IMPORT, namespaces=NAMESPACES): + #LOGGER.info('xml_import_policy = {:s}'.format(str(ET.tostring(xml_import_policy)))) + if xml_import_policy.text is None: continue + iip_ap = {} + add_value_from_tag(iip_ap, 'name', ni_name) + add_value_from_tag(iip_ap, 'import_policy', xml_import_policy) + resource_key = '/network_instance[{:s}]/inter_instance_policies[{:s}]'.format( + iip_ap['name'], iip_ap['import_policy']) + response.append((resource_key, iip_ap)) + + for xml_export_policy in xml_iip_ap.xpath(XPATH_NI_IIP_AP_EXPORT, namespaces=NAMESPACES): + #LOGGER.info('xml_export_policy = {:s}'.format(str(ET.tostring(xml_export_policy)))) + if xml_export_policy.text is None: continue + iip_ap = {} + add_value_from_tag(iip_ap, 'name', ni_name) + add_value_from_tag(iip_ap, 'export_policy', xml_export_policy) + resource_key = '/network_instance[{:s}]/inter_instance_policies[{:s}]'.format( + iip_ap['name'], iip_ap['export_policy']) + response.append((resource_key, iip_ap)) + return response diff --git a/src/device/service/drivers/openconfig/templates/RoutingPolicy.py b/src/device/service/drivers/openconfig/templates/RoutingPolicy.py new file mode 100644 index 0000000000000000000000000000000000000000..aae8483706646801dccf6d3018eb9860209bf52b --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/RoutingPolicy.py @@ -0,0 +1,85 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, logging, lxml.etree as ET +from typing import Any, Dict, List, Tuple +from .Namespace import NAMESPACES +from .Tools import add_value_from_collection, add_value_from_tag + +LOGGER = logging.getLogger(__name__) + +XPATH_POLICY_DEFINITIONS = "//ocrp:routing-policy/ocrp:policy-definitions/ocrp:policy-definition" +XPATH_PD_STATEMENTS = ".//ocrp:statements/ocrp:statement" +XPATH_PD_ST_CONDITIONS = ".//ocrp:conditions/ocbp:bgp-conditions/ocbp:match-ext-community-set" +XPATH_PD_ST_ACTIONS = ".//ocrp:actions" + +XPATH_BGP_EXT_COMMUN_SET = "//ocrp:routing-policy/ocrp:defined-sets/ocbp:bgp-defined-sets/" + \ + "ocbp:ext-community-sets/ocbp:ext-community-set" + +def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: + #LOGGER.info('[RoutePolicy] xml_data = {:s}'.format(str(ET.tostring(xml_data)))) + + response = [] + for xml_policy_definition in xml_data.xpath(XPATH_POLICY_DEFINITIONS, namespaces=NAMESPACES): + #LOGGER.info('xml_policy_definition = {:s}'.format(str(ET.tostring(xml_policy_definition)))) + + policy_definition = {} + + policy_name = xml_policy_definition.find('ocrp:name', namespaces=NAMESPACES) + if policy_name is None or policy_name.text is None: continue + add_value_from_tag(policy_definition, 'policy_name', policy_name) + + resource_key = '/routing_policy/policy_definition[{:s}]'.format(policy_definition['policy_name']) + response.append((resource_key, copy.deepcopy(policy_definition))) + + for xml_statement in xml_policy_definition.xpath(XPATH_PD_STATEMENTS, namespaces=NAMESPACES): + statement_name = xml_statement.find('ocrp:name', namespaces=NAMESPACES) + add_value_from_tag(policy_definition, 'statement_name', statement_name) + + for xml_condition in xml_statement.xpath(XPATH_PD_ST_CONDITIONS, namespaces=NAMESPACES): + ext_community_set_name = xml_condition.find('ocbp:config/ocbp:ext-community-set', namespaces=NAMESPACES) + add_value_from_tag(policy_definition, 'ext_community_set_name', ext_community_set_name) + + match_set_options = xml_condition.find('ocbp:config/ocbp:match-set-options', namespaces=NAMESPACES) + add_value_from_tag(policy_definition, 'match_set_options', match_set_options) + + for xml_action in xml_statement.xpath(XPATH_PD_ST_ACTIONS, namespaces=NAMESPACES): + policy_result = xml_action.find('ocbp:config/ocbp:policy-result', namespaces=NAMESPACES) + add_value_from_tag(policy_definition, 'policy_result', policy_result) + + resource_key = '/routing_policy/policy_definition[{:s}]/statement[{:s}]'.format( + policy_definition['policy_name'], policy_definition['statement_name']) + response.append((resource_key, copy.deepcopy(policy_definition))) + + for xml_bgp_ext_community_set in xml_data.xpath(XPATH_BGP_EXT_COMMUN_SET, namespaces=NAMESPACES): + #LOGGER.info('xml_bgp_ext_community_set = {:s}'.format(str(ET.tostring(xml_bgp_ext_community_set)))) + + bgp_ext_community_set = {} + + ext_community_set_name = xml_bgp_ext_community_set.find('ocbp:ext-community-set-name', namespaces=NAMESPACES) + if ext_community_set_name is None or ext_community_set_name.text is None: continue + add_value_from_tag(bgp_ext_community_set, 'ext_community_set_name', ext_community_set_name) + + resource_key = '/routing_policy/bgp_defined_set[{:s}]'.format(bgp_ext_community_set['ext_community_set_name']) + response.append((resource_key, copy.deepcopy(bgp_ext_community_set))) + + ext_community_member = xml_bgp_ext_community_set.find('ocbp:ext-community-member', namespaces=NAMESPACES) + if ext_community_member is not None and ext_community_member.text is not None: + add_value_from_tag(bgp_ext_community_set, 'ext_community_member', ext_community_member) + + resource_key = '/routing_policy/bgp_defined_set[{:s}][{:s}]'.format( + bgp_ext_community_set['ext_community_set_name'], bgp_ext_community_set['ext_community_member']) + response.append((resource_key, copy.deepcopy(bgp_ext_community_set))) + + return response diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index da1426fd18fb34188cc99ff4a389a285e26a83cd..eb7842ea8d5b62798f08429776700a792f69dc91 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -15,14 +15,17 @@ import json, logging, lxml.etree as ET, re from typing import Any, Dict from jinja2 import Environment, PackageLoader, select_autoescape -from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES) from .EndPoints import parse as parse_endpoints from .Interfaces import parse as parse_interfaces, parse_counters from .NetworkInstances import parse as parse_network_instances +from .RoutingPolicy import parse as parse_routing_policy ALL_RESOURCE_KEYS = [ RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, + RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances RESOURCE_NETWORK_INSTANCES, ] @@ -30,12 +33,14 @@ RESOURCE_KEY_MAPPINGS = { RESOURCE_ENDPOINTS : 'component', RESOURCE_INTERFACES : 'interface', RESOURCE_NETWORK_INSTANCES: 'network_instance', + RESOURCE_ROUTING_POLICIES : 'routing_policy', } RESOURCE_PARSERS = { 'component' : parse_endpoints, 'interface' : parse_interfaces, 'network_instance': parse_network_instances, + 'routing_policy' : parse_routing_policy, 'interfaces/interface/state/counters': parse_counters, } diff --git a/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml index 51bfde07c90e6aa0df4a30a4b32b717ac12de52a..9362c09c6cfebcd1f83b05002f58eda51724b911 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml @@ -18,4 +18,3 @@ {% endif %} - diff --git a/src/device/service/drivers/openconfig/templates/network_instance/inter_instance_policies/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/inter_instance_policies/edit_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..b7c87c7ab13317b5bb2a15c43d241673196bf6d2 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/network_instance/inter_instance_policies/edit_config.xml @@ -0,0 +1,15 @@ +{% if operation is not defined or operation != 'delete' %} + + + {{name}} + + + + {% if import_policy is defined %}{{import_policy}}{% endif%} + {% if export_policy is defined %}{{export_policy}}{% endif%} + + + + + +{% endif %} diff --git a/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..da05d0467605e6cec0c3448cc325ff60dfc7cfc9 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml @@ -0,0 +1,27 @@ + + + {{name}} + + + {{identifier}} + {{protocol_name}} + {% if operation is not defined or operation != 'delete' %} + + {{identifier}} + {{protocol_name}} + true + + {% if identifier=='BGP' %} + + + + {{as}} + + + + {% endif %} + {% endif %} + + + + diff --git a/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml index e7c263d78dea0f786c7e6f5a37e07689efb490b9..46bf5e387789c7efc800ad96ed759748273bed34 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml @@ -3,14 +3,15 @@ {{name}} - oc-pol-types:DIRECTLY_CONNECTED - oc-pol-types:BGP - oc-types:IPV4 + oc-pol-types:{{src_protocol}} + oc-pol-types:{{dst_protocol}} + oc-types:{{address_family}} {% if operation is not defined or operation != 'delete' %} - oc-pol-types:DIRECTLY_CONNECTED - oc-pol-types:BGP - oc-types:IPV4 + oc-pol-types:{{src_protocol}} + oc-pol-types:{{dst_protocol}} + oc-types:{{address_family}} + {% if default_import_policy is defined %}{{default_import_policy}}{% endif %} {% endif %} diff --git a/src/device/service/drivers/openconfig/templates/routing_policy/bgp_defined_set/edit_config.xml b/src/device/service/drivers/openconfig/templates/routing_policy/bgp_defined_set/edit_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..df64606ae5ab434e5e3453f7294db02bb749bdce --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/routing_policy/bgp_defined_set/edit_config.xml @@ -0,0 +1,14 @@ + + + + + + {{ext_community_set_name}} + {% if operation is not defined or operation != 'delete' %} + {% if ext_community_member is defined %} {{ext_community_member}}{% endif %} + {% endif %} + + + + + diff --git a/src/device/service/drivers/openconfig/templates/routing_policy/get.xml b/src/device/service/drivers/openconfig/templates/routing_policy/get.xml new file mode 100644 index 0000000000000000000000000000000000000000..797e970264fa25c2d45f54cb9c0a6cd0b769f29a --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/routing_policy/get.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/edit_config.xml b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/edit_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..c3a31866be5fc072bced339dd7b54f9f92bab290 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/edit_config.xml @@ -0,0 +1,12 @@ + + + + {{policy_name}} + {% if operation is not defined or operation != 'delete' %} + + {{policy_name}} + + {% endif %} + + + diff --git a/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..711067f424b68da0e69913ce01f5133c5cbbfe02 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml @@ -0,0 +1,30 @@ +{% if operation is not defined or operation != 'delete' %} + + + + {{policy_name}} + + + {{statement_name}} + + {{statement_name}} + + + + + {{ext_community_set_name}} + {{match_set_options}} + + + + + + {{policy_result}} + + + + + + + +{% endif %} diff --git a/src/device/tests/test_unitary.py b/src/device/tests/test_unitary.py index 2f84619bbb504a9c55849ccb92f671e1b5e481b4..64c54deb0e784fbf075d7d6973ed01b0708d4867 100644 --- a/src/device/tests/test_unitary.py +++ b/src/device/tests/test_unitary.py @@ -78,14 +78,14 @@ try: except ImportError: ENABLE_P4 = False -ENABLE_EMULATED = False # set to False to disable tests of Emulated devices +#ENABLE_EMULATED = False # set to False to disable tests of Emulated devices #ENABLE_OPENCONFIG = False # set to False to disable tests of OpenConfig devices -ENABLE_TAPI = False # set to False to disable tests of TAPI devices +#ENABLE_TAPI = False # set to False to disable tests of TAPI devices ENABLE_P4 = False # set to False to disable tests of P4 devices (P4 device not available in GitLab) -ENABLE_OPENCONFIG_CONFIGURE = False +ENABLE_OPENCONFIG_CONFIGURE = True ENABLE_OPENCONFIG_MONITOR = True -ENABLE_OPENCONFIG_DECONFIGURE = False +ENABLE_OPENCONFIG_DECONFIGURE = True logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING) @@ -575,6 +575,14 @@ def test_device_openconfig_add_correct( driver : _Driver = device_service.driver_instance_cache.get(DEVICE_OC_UUID) # we know the driver exists now assert driver is not None + device_data = context_client.GetDevice(DeviceId(**DEVICE_OC_ID)) + config_rules = [ + (ConfigActionEnum.Name(config_rule.action), config_rule.resource_key, config_rule.resource_value) + for config_rule in device_data.device_config.config_rules + ] + LOGGER.info('device_data.device_config.config_rules = \n{:s}'.format( + '\n'.join(['{:s} {:s} = {:s}'.format(*config_rule) for config_rule in config_rules]))) + def test_device_openconfig_get( context_client : ContextClient, # pylint: disable=redefined-outer-name