From 3e1df0378bc50c1d67a1c27674f810084bca0bc8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 18:59:10 +0000 Subject: [PATCH 01/34] Device component - gNMI OpenConfig driver: - Updated Component Handler - Updated Interface Handler - Updated InterfaceCounter Handler - Partial update of NetworkInstance Handler - Updated Tools - Added scripts to collect OpenConfig data model and build yang model bindings - Minor cosmetic changes - Updated test_gnmi.py and test_gnmi.sh script - Updated requirements.in --- src/device/requirements.in | 3 +- .../gnmi_openconfig/01-clone-yang-models.sh | 27 +++ .../gnmi_openconfig/02-build-yang-bindings.sh | 106 +++++++++ .../gnmi_openconfig/GnmiSessionHandler.py | 4 +- .../gnmi_openconfig/handlers/Component.py | 42 ++-- .../gnmi_openconfig/handlers/Interface.py | 223 +++++------------- .../handlers/InterfaceCounter.py | 84 +++---- .../handlers/NetworkInstance.py | 9 +- .../drivers/gnmi_openconfig/handlers/Tools.py | 22 +- .../gnmi_openconfig/handlers/__init__.py | 11 +- .../drivers/gnmi_openconfig/tools/Path.py | 6 +- .../drivers/gnmi_openconfig/tools/Value.py | 37 ++- src/device/tests/test_gnmi.py | 50 ++-- test_gnmi.sh | 17 ++ 14 files changed, 366 insertions(+), 275 deletions(-) create mode 100755 src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh create mode 100755 src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh create mode 100755 test_gnmi.sh diff --git a/src/device/requirements.in b/src/device/requirements.in index ece761571..d8a33455e 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -15,6 +15,7 @@ anytree==2.8.0 APScheduler==3.10.1 +bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 Jinja2==3.0.3 @@ -32,7 +33,7 @@ tabulate ipaddress macaddress yattag -pyang +pyang==2.6.* git+https://github.com/robshakir/pyangbind.git websockets==10.4 diff --git a/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh new file mode 100755 index 000000000..fe852f0e1 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig + +rm -rf ${GIT_BASE_PATH} + +OC_PUBLIC_PATH=${GIT_BASE_PATH}/public +mkdir -p ${OC_PUBLIC_PATH} +git clone https://github.com/openconfig/public.git ${OC_PUBLIC_PATH} + +#OC_HERCULES_PATH=${GIT_BASE_PATH}/hercules +#mkdir -p ${OC_HERCULES_PATH} +#git clone https://github.com/openconfig/hercules.git ${OC_HERCULES_PATH} diff --git a/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh new file mode 100755 index 000000000..ed4cf263f --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig +OC_PUBLIC_MODELS_PATH=${GIT_BASE_PATH}/public/release/models +IETF_MODELS_PATH=${GIT_BASE_PATH}/public/third_party/ietf +#OC_HERCULES_MODELS_PATH=${GIT_BASE_PATH}/hercules/yang + +OUT_FOLDER=openconfig +OUT_PATH=${BASE_PATH}/handlers +cd ${OUT_PATH} +export PYBINDPLUGIN=`/usr/bin/env python -c 'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'` + +# -p ${OC_HERCULES_MODELS_PATH}/ +# --split-class-dir openconfig_hercules +pyang --plugindir $PYBINDPLUGIN -p ${OC_PUBLIC_MODELS_PATH}/ -f pybind --split-class-dir ${OUT_FOLDER} \ + ${IETF_MODELS_PATH}/iana-if-type.yang \ + ${IETF_MODELS_PATH}/ietf-interfaces.yang \ + ${IETF_MODELS_PATH}/ietf-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv4-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv6-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match.yang \ + ${OC_PUBLIC_MODELS_PATH}/defined-sets/openconfig-defined-sets.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-aggregate.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ethernet.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ip.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/mpls/openconfig-mpls-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/openconfig-extensions.yang \ + ${OC_PUBLIC_MODELS_PATH}/optical-transport/openconfig-transport-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-common.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-controller-card.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-cpu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-ext.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fabric.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fan.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-healthz.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-integrated-circuit.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-linecard.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-pipeline-counters.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-port.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-psu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-software.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-transceiver.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-elements.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-mem-mgmt.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos.yang \ + ${OC_PUBLIC_MODELS_PATH}/system/openconfig-alarm-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-inet-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l2.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l3.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-errors.yang \ + + +openconfig-aft +openconfig-bgp +openconfig-igmp +openconfig-isis +openconfig-local-routing +openconfig-mpls +openconfig-ospfv2 +openconfig-pcep +openconfig-pim +openconfig-policy-forwarding +openconfig-policy-types +openconfig-routing-policy +openconfig-segment-routing + + + + +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-interfaces.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-chassis.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-linecard.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-node.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-port.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-qos.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules.yang \ diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 04dae4f5f..6f80ee82f 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -132,8 +132,8 @@ class GnmiSessionHandler: #resource_key_tuple[2] = True results.extend(parse(str_path, value)) except Exception as e: # pylint: disable=broad-except - MSG = 'Exception processing notification {:s}' - self._logger.exception(MSG.format(grpc_message_to_json_string(notification))) + MSG = 'Exception processing update {:s}' + self._logger.exception(MSG.format(grpc_message_to_json_string(update))) results.append((str_path, e)) # if validation fails, store the exception #_results = sorted(results.items(), key=lambda x: x[1][0]) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 0b3c1f970..cddf40d56 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,45 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging #, json +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +#pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' - def get_path(self) -> str: return '/components/component' + def get_path(self) -> str: return '/openconfig-platform:components' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_component_list : List[Dict] = json_data.get('component', []) - response = [] - for json_component in json_component_list: - #LOGGER.info('json_component = {:s}'.format(json.dumps(json_component))) - endpoint = {} + oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') + #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) - component_type = json_component.get('state', {}).get('type') - if component_type is None: continue - component_type = component_type.replace('oc-platform-types:', '') - component_type = component_type.replace('openconfig-platform-types:', '') - if component_type not in {'PORT'}: continue - endpoint['type'] = '-' + entries = [] + for component_key, oc_component in oc_components.component.items(): + #LOGGER.info('component_key={:s} oc_component={:s}'.format( + # component_key, pybindJSON.dumps(oc_component, mode='ietf') + #)) - #LOGGER.info('PORT json_component = {:s}'.format(json.dumps(json_component))) + component_name = oc_component.config.name - component_name = json_component.get('name') - if component_name is None: continue + component_type = oc_component.state.type + component_type = component_type.split(':')[-1] + if component_type not in {'PORT'}: continue # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. interface_name = component_name.lower().replace('-port', '') - endpoint['uuid'] = interface_name + endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED : PATH_IF_CTR.format(interface_name, 'in-octets' ), KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED : PATH_IF_CTR.format(interface_name, 'out-octets'), @@ -59,5 +59,7 @@ class ComponentHandler(_Handler): } if len(endpoint) == 0: continue - response.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - return response + + entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 20f79b3c2..77310d51d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -13,15 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' - def get_path(self) -> str: return '/interfaces/interface' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: if_name = str (resource_value['name' ]) # ethernet-1/1 @@ -63,186 +64,88 @@ class InterfaceHandler(_Handler): def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - #LOGGER.info('json_interface = {:s}'.format(json.dumps(json_interface))) + entries = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + #)) interface = {} + interface['name'] = oc_interface.config.name - interface_name = json_interface.get('name') - if interface_name is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue - interface['name'] = interface_name - - CONFIG_FIELDS = ('config', 'openconfig-interface:config', 'oci:config') - json_config : Dict = dict_get_first(json_interface, CONFIG_FIELDS, default={}) - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state : Dict = dict_get_first(json_interface, STATE_FIELDS, default={}) - - interface_type = json_config.get('type') - if interface_type is None: interface_type = json_state.get('type') - if interface_type is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue + interface_type = oc_interface.config.type interface_type = interface_type.replace('ianaift:', '') interface_type = interface_type.replace('iana-if-type:', '') interface['type'] = interface_type - interface_mtu = json_config.get('mtu') - if interface_mtu is None: interface_mtu = json_state.get('mtu') - if interface_mtu is not None: interface['mtu'] = int(interface_mtu) - - interface_enabled = json_config.get('enabled') - if interface_enabled is None: interface_enabled = json_state.get('enabled') - interface['enabled'] = False if interface_enabled is None else bool(interface_enabled) - - interface_management = json_config.get('management') - if interface_management is None: interface_management = json_state.get('management') - interface['management'] = False if interface_management is None else bool(interface_management) + interface['mtu' ] = oc_interface.config.mtu + interface['enabled' ] = oc_interface.config.enabled + interface['description' ] = oc_interface.config.description + interface['admin-status'] = oc_interface.state.admin_status + interface['oper-status' ] = oc_interface.state.oper_status + interface['management' ] = oc_interface.state.management - interface_descr = json_interface.get('config', {}).get('description') - if interface_descr is not None: interface['description'] = interface_descr + entry_interface_key = '/interface[{:s}]'.format(interface['name']) + entries.append((entry_interface_key, interface)) - json_subinterfaces = json_interface.get('subinterfaces', {}) - json_subinterface_list : List[Dict] = json_subinterfaces.get('subinterface', []) - - for json_subinterface in json_subinterface_list: - #LOGGER.info('json_subinterface = {:s}'.format(json.dumps(json_subinterface))) + for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): + #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( + # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') + #)) subinterface = {} + subinterface['index' ] = oc_subinterface.state.index + subinterface['name' ] = oc_subinterface.state.name + subinterface['enabled'] = oc_subinterface.state.enabled - subinterface_index = json_subinterface.get('state', {}).get('index') - if subinterface_index is None: continue - subinterface['index'] = int(subinterface_index) - - subinterface_name = json_subinterface.get('state', {}).get('name') - if subinterface_name is None: continue - subinterface['name'] = subinterface_name - - subinterface_enabled = json_subinterface.get('state', {}).get('enabled', False) - subinterface['enabled'] = bool(subinterface_enabled) - - VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - - # TODO: implement support for multiple IP addresses per subinterface - - IPV4_FIELDS = ('ipv4', 'openconfig-if-ip:ipv4', 'ociip:ipv4') - json_ipv4 = dict_get_first(json_subinterface, IPV4_FIELDS, default={}) - - IPV4_ADDRESSES_FIELDS = ('addresses', 'openconfig-if-ip:addresses', 'ociip:addresses') - json_ipv4_addresses = dict_get_first(json_ipv4, IPV4_ADDRESSES_FIELDS, default={}) - - IPV4_ADDRESS_FIELDS = ('address', 'openconfig-if-ip:address', 'ociip:address') - json_ipv4_address_list : List[Dict] = dict_get_first(json_ipv4_addresses, IPV4_ADDRESS_FIELDS, default=[]) - - #ipv4_addresses = [] - for json_ipv4_address in json_ipv4_address_list: - #LOGGER.info('json_ipv4_address = {:s}'.format(json.dumps(json_ipv4_address))) + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) + entries.append((entry_subinterface_key, subinterface)) - STATE_FIELDS = ('state', 'openconfig-if-ip:state', 'ociip:state') - json_ipv4_address_state = dict_get_first(json_ipv4_address, STATE_FIELDS, default={}) + #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') + #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - #ipv4_address = {} + #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') + #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - #ORIGIN_FIELDS = ('origin', 'openconfig-if-ip:origin', 'ociip:origin') - #ipv4_address_origin = dict_get_first(json_ipv4_address_state, ORIGIN_FIELDS, default={}) - #if ipv4_address_origin is not None: ipv4_address['origin'] = ipv4_address_origin + #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') + #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - IP_FIELDS = ('ip', 'openconfig-if-ip:ip', 'ociip:ip') - ipv4_address_ip = dict_get_first(json_ipv4_address_state, IP_FIELDS) - #if ipv4_address_ip is not None: ipv4_address['address_ip'] = ipv4_address_ip - if ipv4_address_ip is not None: subinterface['address_ip'] = ipv4_address_ip + #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') + #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - PREFIX_FIELDS = ('prefix-length', 'openconfig-if-ip:prefix-length', 'ociip:prefix-length') - ipv4_address_prefix = dict_get_first(json_ipv4_address_state, PREFIX_FIELDS) - #if ipv4_address_prefix is not None: ipv4_address['address_prefix'] = int(ipv4_address_prefix) - if ipv4_address_prefix is not None: subinterface['address_prefix'] = int(ipv4_address_prefix) + #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') + #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) + #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - #if len(ipv4_address) == 0: continue - #ipv4_addresses.append(ipv4_address) + for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): + #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #subinterface['ipv4_addresses'] = ipv4_addresses - - if len(subinterface) == 0: continue - resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) - response.append((resource_key, subinterface)) - - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) - - return response - - def parse_counters(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse_counters] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) - - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse_counters] json_interface = {:s}'.format(json.dumps(json_interface))) - - interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) + address_ipv4 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) + entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) + entries.append((entry_address_ipv4_key, address_ipv4)) - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) + for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): + #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #LOGGER.info('[parse_counters] interface = {:s}'.format(str(interface))) + address_ipv6 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) + entries.append((entry_address_ipv6_key, address_ipv6)) - return response + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index a45dc9e7f..1c2cfc17a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -14,67 +14,51 @@ import json, logging from typing import Any, Dict, List, Tuple +import pyangbind.lib.pybindJSON as pybindJSON +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) +#pylint: disable=abstract-method class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' - def get_path(self) -> str: return '/interfaces/interface/state/counters' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse] json_interface = {:s}'.format(json.dumps(json_interface))) + counters = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + )) interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) - - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) - - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) - - #LOGGER.info('[parse] interface = {:s}'.format(str(interface))) + interface['name'] = oc_interface.name + + interface_counters = oc_interface.state.counters + interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts + interface['in-discards' ] = interface_counters.in_discards + interface['in-errors' ] = interface_counters.in_errors + interface['in-fcs-errors' ] = interface_counters.in_fcs_errors + interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts + interface['in-octets' ] = interface_counters.in_octets + interface['in-pkts' ] = interface_counters.in_pkts + interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts + interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts + interface['out-discards' ] = interface_counters.out_discards + interface['out-errors' ] = interface_counters.out_errors + interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts + interface['out-octets' ] = interface_counters.out_octets + interface['out-pkts' ] = interface_counters.out_pkts + interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + + LOGGER.info('interface = {:s}'.format(str(interface))) if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + counters.append(('/interface[{:s}]'.format(interface['name']), interface)) - return response + return counters diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index aed821a06..c29ed263a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -13,14 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' - def get_path(self) -> str: return '/network-instances/network-instance' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: ni_name = str(resource_value['name']) # test-svc @@ -58,5 +60,10 @@ class NetworkInstanceHandler(_Handler): } def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) response = [] return response + +openconfig-network-instance:network-instance \ No newline at end of file diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 30343ac28..8cf704e29 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -23,8 +23,20 @@ def get_schema(resource_key : str): resource_key = RE_REMOVE_NAMESPACES.sub('/', resource_key) return resource_key -def dict_get_first(d : Dict, field_names : Iterable[str], default=None) -> Any: - for field_name in field_names: - if field_name not in d: continue - return d[field_name] +def container_get_first( + container : Dict[str, Any], key_name : str, namespace : Optional[str]=None, namespaces : Iterable[str]=tuple(), + default : Optional[Any] = None +) -> Any: + value = container.get(key_name) + if value is not None: return value + + if namespace is not None: + if len(namespaces) > 0: + raise Exception('At maximum, one of namespace or namespaces can be specified') + namespaces = (namespace,) + + for namespace in namespaces: + namespace_key_name = '{:s}:{:s}'.format(namespace, key_name) + if namespace_key_name in container: return container[namespace_key_name] + return default diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 39cd7c66a..6d54ef28d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -46,9 +46,10 @@ RESOURCE_KEY_MAPPER = { } PATH_MAPPER = { - '/components' : comph.get_path(), - '/interfaces' : ifaceh.get_path(), - '/network-instances' : nih.get_path(), + '/components' : comph.get_path(), + '/components/component' : comph.get_path(), + '/interfaces' : ifaceh.get_path(), + '/network-instances' : nih.get_path(), } RESOURCE_KEY_TO_HANDLER = { @@ -88,9 +89,9 @@ def get_handler( path_schema = PATH_MAPPER.get(path_schema, path_schema) handler = PATH_TO_HANDLER.get(path_schema) if handler is None and raise_if_not_found: - MSG = 'Handler not found: resource_key={:s} resource_key_schema={:s}' + MSG = 'Handler not found: path={:s} path_schema={:s}' # pylint: disable=broad-exception-raised - raise Exception(MSG.format(str(resource_key), str(resource_key_schema))) + raise Exception(MSG.format(str(path), str(path_schema))) return handler def get_path(resource_key : str) -> str: diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Path.py b/src/device/service/drivers/gnmi_openconfig/tools/Path.py index 40ab28dc6..2d6dc1e74 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Path.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Path.py @@ -19,8 +19,8 @@ from ..gnmi.gnmi_pb2 import Path, PathElem RE_PATH_SPLIT = re.compile(r'/(?=(?:[^\[\]]|\[[^\[\]]+\])*$)') RE_PATH_KEYS = re.compile(r'\[(.*?)\]') -def path_from_string(path='/'): - if not path: return Path(elem=[]) +def path_from_string(path='/'): #, origin='openconfig' + if not path: return Path(elem=[]) #, origin=origin if path[0] == '/': if path[-1] == '/': @@ -40,7 +40,7 @@ def path_from_string(path='/'): dict_keys = dict(x.split('=', 1) for x in elem_keys) path.append(PathElem(name=elem_name, key=dict_keys)) - return Path(elem=path) + return Path(elem=path) #, origin=origin def path_to_string(path : Path) -> str: path_parts = list() diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 4797930a1..9933cb858 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -13,9 +13,36 @@ # limitations under the License. import base64, json -from typing import Any +from typing import Any, Dict, List, Union from ..gnmi.gnmi_pb2 import TypedValue +REMOVE_NAMESPACES = ( + 'arista-intf-augments', + 'arista-netinst-augments', + 'openconfig-hercules-platform', +) + +def remove_fields(key : str) -> bool: + parts = key.split(':') + if len(parts) == 1: return False + namespace = parts[0].lower() + return namespace in REMOVE_NAMESPACES + +def recursive_remove_keys(container : Union[Dict, List, Any]) -> None: + if isinstance(container, dict): + remove_keys = [ + key + for key in container.keys() + if remove_fields(key) + ] + for key in remove_keys: + container.pop(key, None) + for value in container.values(): + recursive_remove_keys(value) + elif isinstance(container, list): + for value in container: + recursive_remove_keys(value) + def decode_value(value : TypedValue) -> Any: encoding = value.WhichOneof('value') if encoding == 'json_val': @@ -31,9 +58,13 @@ def decode_value(value : TypedValue) -> Any: raise NotImplementedError() #return value elif encoding == 'json_ietf_val': - value : str = value.json_ietf_val + str_value : str = value.json_ietf_val.decode('UTF-8') try: - return json.loads(value) + # Cleanup and normalize the records according to OpenConfig + str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + json_value = json.loads(str_value) + recursive_remove_keys(json_value) + return json_value except json.decoder.JSONDecodeError: # Assume is Base64-encoded b_b64_value = value.encode('UTF-8') diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py index 50c915582..684b9f4c3 100644 --- a/src/device/tests/test_gnmi.py +++ b/src/device/tests/test_gnmi.py @@ -16,9 +16,9 @@ import logging, os, sys, time from typing import Dict, Tuple os.environ['DEVICE_EMULATED_ONLY'] = 'YES' from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -#from device.service.driver_api._Driver import ( -# RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -#) +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -58,21 +58,21 @@ def main(): driver_settings = { 'protocol': 'gnmi', 'username': 'admin', - 'password': 'NokiaSrl1!', - 'use_tls' : True, + 'password': 'admin', + 'use_tls' : False, } - driver = GnmiOpenConfigDriver('172.100.100.102', 57400, **driver_settings) + driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) driver.Connect() #resources_to_get = [] #resources_to_get = [RESOURCE_ENDPOINTS] #resources_to_get = [RESOURCE_INTERFACES] - #resources_to_get = [RESOURCE_NETWORK_INSTANCES] + resources_to_get = [RESOURCE_NETWORK_INSTANCES] #resources_to_get = [RESOURCE_ROUTING_POLICIES] #resources_to_get = [RESOURCE_SERVICES] - #LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - #results_getconfig = driver.GetConfig(resources_to_get) - #LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) #resources_to_set = [ # network_instance('test-svc', 'L3VRF'), @@ -90,21 +90,21 @@ def main(): #results_setconfig = driver.SetConfig(resources_to_set) #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - resources_to_delete = [ - #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - - #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - - interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - - network_instance('20f66fb5', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + #resources_to_delete = [ + # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), + # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), + # + # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), + # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), + # + # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), + # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), + # + # #network_instance('20f66fb5', 'L3VRF'), + #] + #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + #results_deleteconfig = driver.DeleteConfig(resources_to_delete) + #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) time.sleep(1) diff --git a/test_gnmi.sh b/test_gnmi.sh new file mode 100755 index 000000000..d1fe36969 --- /dev/null +++ b/test_gnmi.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export PYTHONPATH=./src +python -m device.tests.test_gnmi -- GitLab From a345795694cb86ccbdace416441b32cffb42afe8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 19:02:09 +0000 Subject: [PATCH 02/34] DataPlane in a box: - Added management scripts - Added ContainerLab scenario descriptor - Added README.md - Added example TFS descriptors - Added links.txt --- dataplane-in-a-box/.gitignore | 2 + dataplane-in-a-box/README.md | 121 ++++++++++++++++ dataplane-in-a-box/arista.clab.yml | 54 ++++++++ dataplane-in-a-box/clab-deploy.sh | 17 +++ dataplane-in-a-box/clab-destroy.sh | 18 +++ dataplane-in-a-box/clab-inspect.sh | 17 +++ dataplane-in-a-box/clab-load-image.sh | 19 +++ dataplane-in-a-box/clab-pull-images.sh | 18 +++ dataplane-in-a-box/dc-2-dc-l3-service.json | 37 +++++ dataplane-in-a-box/deploy_specs.sh | 154 +++++++++++++++++++++ dataplane-in-a-box/links.json | 136 ++++++++++++++++++ dataplane-in-a-box/links.txt | 8 ++ dataplane-in-a-box/topology.json | 91 ++++++++++++ 13 files changed, 692 insertions(+) create mode 100644 dataplane-in-a-box/.gitignore create mode 100644 dataplane-in-a-box/README.md create mode 100644 dataplane-in-a-box/arista.clab.yml create mode 100755 dataplane-in-a-box/clab-deploy.sh create mode 100755 dataplane-in-a-box/clab-destroy.sh create mode 100755 dataplane-in-a-box/clab-inspect.sh create mode 100755 dataplane-in-a-box/clab-load-image.sh create mode 100755 dataplane-in-a-box/clab-pull-images.sh create mode 100644 dataplane-in-a-box/dc-2-dc-l3-service.json create mode 100755 dataplane-in-a-box/deploy_specs.sh create mode 100644 dataplane-in-a-box/links.json create mode 100644 dataplane-in-a-box/links.txt create mode 100644 dataplane-in-a-box/topology.json diff --git a/dataplane-in-a-box/.gitignore b/dataplane-in-a-box/.gitignore new file mode 100644 index 000000000..5de716bdd --- /dev/null +++ b/dataplane-in-a-box/.gitignore @@ -0,0 +1,2 @@ +clab-arista/ +.arista.clab.yml.bak diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md new file mode 100644 index 000000000..45e5dc5e0 --- /dev/null +++ b/dataplane-in-a-box/README.md @@ -0,0 +1,121 @@ +# DataPlane-in-a-Box - Control an Emulated DataPlane through TeraFlowSDN + +## Emulated DataPlane Deployment +- ContainerLab +- Scenario +- Descriptor + +## TeraFlowSDN Deployment +```bash +cd ~/tfs-ctrl +source dataplane-in-a-box/deploy_specs.sh +./deploy/all.sh +``` + +# ContainerLab - Arista cEOS - Commands + +## Download and install ContainerLab +```bash +sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 +``` + +## Deploy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml +``` + +## Inspect scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml +``` + +## Destroy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak +``` + +## Access cEOS Bash +```bash +docker exec -it clab-arista-wan1 bash +``` + +## Access cEOS CLI +```bash +docker exec -it clab-arista-wan1 Cli +``` + +## Configure ContainerLab clients +```bash +docker exec -it clab-arista-client1 bash + ip address add 192.168.1.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.1.1 + ip route add 192.168.3.0/24 via 192.168.1.1 + ping 192.168.2.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client2 bash + ip address add 192.168.2.10/24 dev eth1 + ip route add 192.168.1.0/24 via 192.168.2.1 + ip route add 192.168.3.0/24 via 192.168.2.1 + ping 192.168.1.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client3 bash + ip address add 192.168.3.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.3.1 + ip route add 192.168.3.0/24 via 192.168.3.1 + ping 192.168.2.10 + ping 192.168.3.10 +``` + +## Install gNMIc +```bash +sudo bash -c "$(curl -sL https://get-gnmic.kmrd.dev)" +``` + +## gNMI Capabilities request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure capabilities +``` + +## gNMI Get request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path / > wan1.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /interfaces/interface > wan1-ifaces.json +``` + +## gNMI Set request +```bash +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --update-path /system/config/hostname --update-value srl11 +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /system/config/hostname +``` + +## Subscribe request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf subscribe --path /interfaces/interface[name=Management0]/state/ + +# In another terminal, you can generate traffic opening SSH connection +ssh admin@clab-arista-wan1 +``` + +# Check configurations done: +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/network-instances' > wan1-nis.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/interfaces' > wan1-ifs.json +``` + +# Delete elements: +```bash +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/network-instances/network-instance[name=b19229e8]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/1]/subinterfaces/subinterface[index=0]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/2]/subinterfaces/subinterface[index=0]' +``` + +# Run gNMI Driver in standalone mode (advanced) +```bash +PYTHONPATH=./src python -m src.device.tests.test_gnmi +``` diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml new file mode 100644 index 000000000..9a8bff73f --- /dev/null +++ b/dataplane-in-a-box/arista.clab.yml @@ -0,0 +1,54 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TFS - Arista devices + Linux clients + +name: arista + +mgmt: + network: mgmt-net + ipv4-subnet: 172.20.20.0/24 + +topology: + kinds: + arista_ceos: + kind: arista_ceos + image: ceos:4.30.4M + linux: + kind: linux + image: ghcr.io/hellt/network-multitool:latest + + nodes: + wan1: + kind: arista_ceos + mgmt-ipv4: 172.20.20.101 + ports: [6001:6030] + wan2: + kind: arista_ceos + mgmt-ipv4: 172.20.20.102 + ports: [6002:6030] + + client1: + kind: linux + mgmt-ipv4: 172.20.20.201 + ports: [2201:22] + client2: + kind: linux + mgmt-ipv4: 172.20.20.202 + ports: [2202:22] + + links: + - endpoints: ["wan1:eth1", "wan2:eth1"] + - endpoints: ["client1:eth1", "wan1:eth10"] + - endpoints: ["client2:eth1", "wan2:eth10"] diff --git a/dataplane-in-a-box/clab-deploy.sh b/dataplane-in-a-box/clab-deploy.sh new file mode 100755 index 000000000..2b8e49a07 --- /dev/null +++ b/dataplane-in-a-box/clab-deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-destroy.sh b/dataplane-in-a-box/clab-destroy.sh new file mode 100755 index 000000000..4030239dc --- /dev/null +++ b/dataplane-in-a-box/clab-destroy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak diff --git a/dataplane-in-a-box/clab-inspect.sh b/dataplane-in-a-box/clab-inspect.sh new file mode 100755 index 000000000..02024ec47 --- /dev/null +++ b/dataplane-in-a-box/clab-inspect.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh new file mode 100755 index 000000000..87e666422 --- /dev/null +++ b/dataplane-in-a-box/clab-load-image.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download image from Arista account > software downloads + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh new file mode 100755 index 000000000..8f2805c6b --- /dev/null +++ b/dataplane-in-a-box/clab-pull-images.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker pull ghcr.io/hellt/network-multitool:latest +#docker pull ghcr.io/nokia/srlinux:23.7.2 +#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/dc-2-dc-l3-service.json new file mode 100644 index 000000000..cb9ef972e --- /dev/null +++ b/dataplane-in-a-box/dc-2-dc-l3-service.json @@ -0,0 +1,37 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + }, + "service_type": 1, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, + {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}}, + + {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}} + ]} + } + ] +} diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh new file mode 100755 index 000000000..1a978e3a9 --- /dev/null +++ b/dataplane-in-a-box/deploy_specs.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" + +# Uncomment to activate Monitoring +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json new file mode 100644 index 000000000..832a24fdd --- /dev/null +++ b/dataplane-in-a-box/links.json @@ -0,0 +1,136 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/links.txt b/dataplane-in-a-box/links.txt new file mode 100644 index 000000000..a61ad5398 --- /dev/null +++ b/dataplane-in-a-box/links.txt @@ -0,0 +1,8 @@ +https://containerlab.dev/manual/multi-node/#exposing-services +https://containerlab.dev/manual/multi-node/#bridging +https://containerlab.dev/manual/kinds/bridge/ +https://containerlab.dev/lab-examples/ext-bridge/ + +https://containerlab.dev/manual/kinds/ceos/ +https://containerlab.dev/lab-examples/srl-ceos/#__tabbed_2_2 +https://github.com/srl-labs/containerlab/blob/main/lab-examples/srlceos01/srlceos01.clab.yml diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/topology.json new file mode 100644 index 000000000..42752235d --- /dev/null +++ b/dataplane-in-a-box/topology.json @@ -0,0 +1,91 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + } + ], + "links": [] +} -- GitLab From 48c2413c0b6bd58a4a1bc571415175aed133120f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:25:34 +0000 Subject: [PATCH 03/34] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Added unitary tests and scripts - Enhanced reporting of capabilities - Migrated Component and Interface code to libyang - Migrating NetworkInstance code to libyang - Disabled unneeded log messages - Temporarily disabled telemetry - Added LibYang-based YANG handler - Added helper methods --- ...un_tests_locally-device-gnmi-openconfig.sh | 25 + .../gnmi_openconfig/GnmiSessionHandler.py | 65 +- .../gnmi_openconfig/handlers/Component.py | 39 +- .../gnmi_openconfig/handlers/Interface.py | 276 ++++---- .../handlers/NetworkInstance.py | 145 ++++- .../drivers/gnmi_openconfig/handlers/Tools.py | 21 +- .../gnmi_openconfig/handlers/YangHandler.py | 109 ++++ .../gnmi_openconfig/handlers/_Handler.py | 9 +- .../gnmi_openconfig/handlers/__init__.py | 24 +- .../gnmi_openconfig/tools/Capabilities.py | 17 +- .../drivers/gnmi_openconfig/tools/Value.py | 2 +- src/device/tests/test_gnmi.py | 115 ---- .../tests/test_unitary_gnmi_openconfig.py | 616 ++++++++++++++++++ 13 files changed, 1149 insertions(+), 314 deletions(-) create mode 100755 scripts/run_tests_locally-device-gnmi-openconfig.sh create mode 100644 src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py delete mode 100644 src/device/tests/test_gnmi.py create mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh new file mode 100755 index 000000000..d81684da1 --- /dev/null +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc + +# Run unitary tests and analyze coverage of code at same time +# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + device/tests/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 6f80ee82f..d9f73c958 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -19,12 +19,13 @@ from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk from .gnmi.gnmi_pb2_grpc import gNMIStub from .gnmi.gnmi_pb2 import Encoding, GetRequest, SetRequest, UpdateResult # pylint: disable=no-name-in-module from .handlers import ALL_RESOURCE_KEYS, compose, get_path, parse -from .tools.Capabilities import get_supported_encodings +from .handlers.YangHandler import YangHandler +from .tools.Capabilities import check_capabilities from .tools.Channel import get_grpc_channel from .tools.Path import path_from_string, path_to_string #, compose_path from .tools.Subscriptions import Subscriptions from .tools.Value import decode_value #, value_exists -from .MonitoringThread import MonitoringThread +#from .MonitoringThread import MonitoringThread class GnmiSessionHandler: def __init__(self, address : str, port : int, settings : Dict, logger : logging.Logger) -> None: @@ -39,12 +40,20 @@ class GnmiSessionHandler: self._use_tls = settings.get('use_tls', False) self._channel : Optional[grpc.Channel] = None self._stub : Optional[gNMIStub] = None - self._monit_thread = None - self._supported_encodings = None + self._yang_handler = YangHandler() + #self._monit_thread = None self._subscriptions = Subscriptions() self._in_subscriptions = queue.Queue() self._out_samples = queue.Queue() + def __del__(self) -> None: + self._logger.warning('Destroying YangValidator...') + self._logger.warning('yang_validator.data:') + for path, dnode in self._yang_handler.get_data_paths().items(): + self._logger.warning(' {:s}: {:s}'.format(str(path), json.dumps(dnode.print_dict()))) + self._yang_handler.destroy() + self._logger.warning('DONE') + @property def subscriptions(self): return self._subscriptions @@ -58,18 +67,17 @@ class GnmiSessionHandler: with self._lock: self._channel = get_grpc_channel(self._address, self._port, self._use_tls, self._logger) self._stub = gNMIStub(self._channel) - self._supported_encodings = get_supported_encodings( - self._stub, self._username, self._password, timeout=120) - self._monit_thread = MonitoringThread( - self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) - self._monit_thread.start() + check_capabilities(self._stub, self._username, self._password, timeout=120) + #self._monit_thread = MonitoringThread( + # self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) + #self._monit_thread.start() self._connected.set() def disconnect(self): if not self._connected.is_set(): return with self._lock: - self._monit_thread.stop() - self._monit_thread.join() + #self._monit_thread.stop() + #self._monit_thread.join() self._channel.close() self._connected.clear() @@ -87,9 +95,9 @@ class GnmiSessionHandler: str_resource_name = 'resource_key[#{:d}]'.format(i) try: chk_string(str_resource_name, resource_key, allow_empty=False) - self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) + #self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) str_path = get_path(resource_key) - self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) + #self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) get_request.path.append(path_from_string(str_path)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception parsing {:s}: {:s}' @@ -130,7 +138,7 @@ class GnmiSessionHandler: value = decode_value(update.val) #resource_key_tuple[1] = value #resource_key_tuple[2] = True - results.extend(parse(str_path, value)) + results.extend(parse(str_path, value, self._yang_handler)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception processing update {:s}' self._logger.exception(MSG.format(grpc_message_to_json_string(update))) @@ -159,17 +167,17 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=False) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=False) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_list = set_request.update #if exists else set_request.replace set_request_entry = set_request_list.add() set_request_entry.path.CopyFrom(path_from_string(str_path)) @@ -228,18 +236,19 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple #if not exists: continue if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=True) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + # pylint: disable=unused-variable + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=True) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_entry = set_request.delete.add() set_request_entry.CopyFrom(path_from_string(str_path)) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index cddf40d56..73728192f 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,37 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging #, json -import pyangbind.lib.pybindJSON as pybindJSON +import json, logging # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}' #pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' def get_path(self) -> str: return '/openconfig-platform:components' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) - oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') - #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) + yang_components_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_components_path, json_data, fmt='json') entries = [] - for component_key, oc_component in oc_components.component.items(): - #LOGGER.info('component_key={:s} oc_component={:s}'.format( - # component_key, pybindJSON.dumps(oc_component, mode='ietf') - #)) + for component in json_data_valid['components']['component']: + LOGGER.debug('component={:s}'.format(str(component))) - component_name = oc_component.config.name + component_name = component['name'] + #component_config = component.get('config', {}) - component_type = oc_component.state.type + #yang_components : libyang.DContainer = yang_handler.get_data_path(yang_components_path) + #yang_component_path = 'component[name="{:s}"]'.format(component_name) + #yang_component : libyang.DContainer = yang_components.create_path(yang_component_path) + #yang_component.merge_data_dict(component, strict=True, validate=False) + + component_state = component.get('state', {}) + component_type = component_state.get('type') + if component_type is None: continue component_type = component_type.split(':')[-1] if component_type not in {'PORT'}: continue @@ -58,8 +65,6 @@ class ComponentHandler(_Handler): KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED: PATH_IF_CTR.format(interface_name, 'out-pkts' ), } - if len(endpoint) == 0: continue - entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 77310d51d..f28cdcf36 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -24,9 +24,11 @@ class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' def get_path(self) -> str: return '/openconfig-interfaces:interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - if_name = str (resource_value['name' ]) # ethernet-1/1 - sif_index = int (resource_value.get('sub_if_index' , 0 )) # 0 + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + if_name = get_str(resource_value, 'name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sub_if_index', 0) # 0 if delete: PATH_TMPL = '/interfaces/interface[name={:s}]/subinterfaces/subinterface[index={:d}]' @@ -34,118 +36,166 @@ class InterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - if_enabled = bool(resource_value.get('enabled' , True)) # True/False - sif_enabled = bool(resource_value.get('sub_if_enabled' , True)) # True/False - sif_ipv4_enabled = bool(resource_value.get('sub_if_ipv4_enabled', True)) # True/False - sif_ipv4_address = str (resource_value['sub_if_ipv4_address' ]) # 172.16.0.1 - sif_ipv4_prefix = int (resource_value['sub_if_ipv4_prefix' ]) # 24 + if_enabled = get_bool(resource_value, 'enabled', True) # True/False + sif_enabled = get_bool(resource_value, 'sub_if_enabled', True) # True/False + sif_vlan_id = get_int (resource_value, 'sif_vlan_id', ) # 127 + sif_ipv4_enabled = get_bool(resource_value, 'sub_if_ipv4_enabled', True) # True/False + sif_ipv4_address = get_str (resource_value, 'sub_if_ipv4_address' ) # 172.16.0.1 + sif_ipv4_prefix = get_int (resource_value, 'sub_if_ipv4_prefix' ) # 24 + + yang_ifs : libyang.DContainer = yang_handler.get_data_path('/openconfig-interfaces:interfaces') + yang_if_path = 'interface[name="{:s}"]'.format(if_name) + yang_if : libyang.DContainer = yang_ifs.create_path(yang_if_path) + yang_if.create_path('config/name', if_name ) + if if_enabled is not None: yang_if.create_path('config/enabled', if_enabled) + + yang_sifs : libyang.DContainer = yang_if.create_path('subinterfaces') + yang_sif_path = 'subinterface[index="{:d}"]'.format(sif_index) + yang_sif : libyang.DContainer = yang_sifs.create_path(yang_sif_path) + yang_sif.create_path('config/index', sif_index) + if sif_enabled is not None: yang_sif.create_path('config/enabled', sif_enabled) + + if sif_vlan_id is not None: + yang_subif_vlan : libyang.DContainer = yang_sif.create_path('openconfig-vlan:vlan') + yang_subif_vlan.create_path('match/single-tagged/config/vlan-id', sif_vlan_id) + + yang_ipv4 : libyang.DContainer = yang_sif.create_path('openconfig-if-ip:ipv4') + if sif_ipv4_enabled is not None: yang_ipv4.create_path('config/enabled', sif_ipv4_enabled) + + if sif_ipv4_address is not None: + yang_ipv4_addrs : libyang.DContainer = yang_ipv4.create_path('addresses') + yang_ipv4_addr_path = 'address[ip="{:s}"]'.format(sif_ipv4_address) + yang_ipv4_addr : libyang.DContainer = yang_ipv4_addrs.create_path(yang_ipv4_addr_path) + yang_ipv4_addr.create_path('config/ip', sif_ipv4_address) + yang_ipv4_addr.create_path('config/prefix-length', sif_ipv4_prefix ) str_path = '/interfaces/interface[name={:s}]'.format(if_name) - str_data = json.dumps({ - 'name': if_name, - 'config': {'name': if_name, 'enabled': if_enabled}, - 'subinterfaces': { - 'subinterface': { - 'index': sif_index, - 'config': {'index': sif_index, 'enabled': sif_enabled}, - 'ipv4': { - 'config': {'enabled': sif_ipv4_enabled}, - 'addresses': { - 'address': { - 'ip': sif_ipv4_address, - 'config': {'ip': sif_ipv4_address, 'prefix_length': sif_ipv4_prefix}, - } - } - } - } - } - }) + str_data = yang_if.print_mem('json') + json_data = json.loads(str_data) + json_data = json_data['openconfig-interfaces:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') entries = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - #)) - - interface = {} - interface['name'] = oc_interface.config.name - - interface_type = oc_interface.config.type - interface_type = interface_type.replace('ianaift:', '') - interface_type = interface_type.replace('iana-if-type:', '') - interface['type'] = interface_type - - interface['mtu' ] = oc_interface.config.mtu - interface['enabled' ] = oc_interface.config.enabled - interface['description' ] = oc_interface.config.description - interface['admin-status'] = oc_interface.state.admin_status - interface['oper-status' ] = oc_interface.state.oper_status - interface['management' ] = oc_interface.state.management - - entry_interface_key = '/interface[{:s}]'.format(interface['name']) - entries.append((entry_interface_key, interface)) - - for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): - #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( - # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') - #)) - - subinterface = {} - subinterface['index' ] = oc_subinterface.state.index - subinterface['name' ] = oc_subinterface.state.name - subinterface['enabled'] = oc_subinterface.state.enabled - - entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) - entries.append((entry_subinterface_key, subinterface)) - - #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): - #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv4 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) - entries.append((entry_address_ipv4_key, address_ipv4)) - - for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): - #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv6 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) - entries.append((entry_address_ipv6_key, address_ipv6)) + for interface in json_data_valid['interfaces']['interface']: + LOGGER.debug('interface={:s}'.format(str(interface))) + + interface_name = interface['name'] + interface_config = interface.get('config', {}) + + #yang_interfaces : libyang.DContainer = yang_handler.get_data_path(yang_interfaces_path) + #yang_interface_path = 'interface[name="{:s}"]'.format(interface_name) + #yang_interface : libyang.DContainer = yang_interfaces.create_path(yang_interface_path) + #yang_interface.merge_data_dict(interface, strict=True, validate=False) + + interface_state = interface.get('state', {}) + interface_type = interface_state.get('type') + if interface_type is None: continue + interface_type = interface_type.split(':')[-1] + if interface_type not in {'ethernetCsmacd'}: continue + + _interface = { + 'name' : interface_name, + 'type' : interface_type, + 'mtu' : interface_state['mtu'], + 'ifindex' : interface_state['ifindex'], + 'admin-status' : interface_state['admin-status'], + 'oper-status' : interface_state['oper-status'], + 'management' : interface_state['management'], + } + if 'description' in interface_config: + _interface['description'] = interface_config['description'] + if 'enabled' in interface_config: + _interface['enabled'] = interface_config['enabled'] + if 'hardware-port' in interface_state: + _interface['hardware-port'] = interface_state['hardware-port'] + if 'transceiver' in interface_state: + _interface['transceiver'] = interface_state['transceiver'] + + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) + + if interface_type == 'ethernetCsmacd': + ethernet_state = interface['ethernet']['state'] + + _ethernet = { + 'mac-address' : ethernet_state['mac-address'], + 'hw-mac-address' : ethernet_state['hw-mac-address'], + 'port-speed' : ethernet_state['port-speed'].split(':')[-1], + 'negotiated-port-speed' : ethernet_state['negotiated-port-speed'].split(':')[-1], + } + entry_ethernet_key = '{:s}/ethernet'.format(entry_interface_key) + entries.append((entry_ethernet_key, _ethernet)) + + subinterfaces = interface.get('subinterfaces', {}).get('subinterface', []) + for subinterface in subinterfaces: + LOGGER.debug('subinterface={:s}'.format(str(subinterface))) + + subinterface_index = subinterface['index'] + subinterface_state = subinterface.get('state', {}) + + _subinterface = {'index': subinterface_index} + if 'name' in subinterface_state: + _subinterface['name'] = subinterface_state['name'] + if 'enabled' in subinterface_state: + _subinterface['enabled'] = subinterface_state['enabled'] + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface_index) + entries.append((entry_subinterface_key, _subinterface)) + + if 'vlan' in subinterface: + vlan = subinterface['vlan'] + vlan_match = vlan['match'] + + single_tagged = vlan_match.pop('single-tagged', None) + if single_tagged is not None: + single_tagged_config = single_tagged['config'] + vlan_id = single_tagged_config['vlan-id'] + + _vlan = {'vlan_id': vlan_id} + entry_vlan_key = '{:s}/vlan[single:{:s}]'.format(entry_subinterface_key, vlan_id) + entries.append((entry_vlan_key, _vlan)) + + if len(vlan_match) > 0: + raise Exception('Unsupported VLAN schema: {:s}'.format(str(vlan))) + + ipv4_addresses = subinterface.get('ipv4', {}).get('addresses', {}).get('address', []) + for ipv4_address in ipv4_addresses: + LOGGER.debug('ipv4_address={:s}'.format(str(ipv4_address))) + + ipv4_address_ip = ipv4_address['ip'] + ipv4_address_state = ipv4_address.get('state', {}) + + _ipv4_address = {'ip': ipv4_address_ip} + if 'origin' in ipv4_address_state: + _ipv4_address['origin'] = ipv4_address_state['origin'] + if 'prefix-length' in ipv4_address_state: + _ipv4_address['prefix'] = ipv4_address_state['prefix-length'] + + entry_ipv4_address_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, ipv4_address_ip) + entries.append((entry_ipv4_address_key, _ipv4_address)) + + ipv6_addresses = subinterface.get('ipv6', {}).get('addresses', {}).get('address', []) + for ipv6_address in ipv6_addresses: + LOGGER.debug('ipv6_address={:s}'.format(str(ipv6_address))) + + ipv6_address_ip = ipv6_address['ip'] + ipv6_address_state = ipv6_address.get('state', {}) + + _ipv6_address = {'ip': ipv6_address_ip} + if 'origin' in ipv6_address_state: + _ipv6_address['origin'] = ipv6_address_state['origin'] + if 'prefix-length' in ipv6_address_state: + _ipv6_address['prefix'] = ipv6_address_state['prefix-length'] + + entry_ipv6_address_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, ipv6_address_ip) + entries.append((entry_ipv6_address_key, _ipv6_address)) return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index c29ed263a..0b4d15745 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,20 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging +import operator from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +MAP_NETWORK_INSTANCE_TYPE = { + # special routing instance; acts as default/global routing instance for a network device + 'DEFAULT': 'openconfig-network-instance-types:DEFAULT_INSTANCE', + + # private L3-only routing instance; formed of one or more RIBs + 'L3VRF': 'openconfig-network-instance-types:L3VRF', + + # private L2-only switch instance; formed of one or more L2 forwarding tables + 'L2VSI': 'openconfig-network-instance-types:L2VSI', + + # private L2-only forwarding instance; point to point connection between two endpoints + 'L2P2P': 'openconfig-network-instance-types:L2P2P', + + # private Layer 2 and Layer 3 forwarding instance + 'L2L3': 'openconfig-network-instance-types:L2L3', +} + class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' def get_path(self) -> str: return '/openconfig-network-instance:network-instances' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name']) # test-svc + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -33,15 +53,11 @@ class NetworkInstanceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - ni_type = str(resource_value['type']) # L3VRF / L2VSI / ... + ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... + ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # not works: [FailedPrecondition] unsupported identifier 'DIRECTLY_CONNECTED' - #protocols = [self._compose_directly_connected()] + # 'DIRECTLY_CONNECTED' is implicitly added - MAP_OC_NI_TYPE = { - 'L3VRF': 'openconfig-network-instance-types:L3VRF', - } - ni_type = MAP_OC_NI_TYPE.get(ni_type, ni_type) str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) str_data = json.dumps({ @@ -51,19 +67,92 @@ class NetworkInstanceHandler(_Handler): }) return str_path, str_data - def _compose_directly_connected(self, name=None, enabled=True) -> Dict: - identifier = 'DIRECTLY_CONNECTED' - if name is None: name = 'DIRECTLY_CONNECTED' - return { - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': enabled}, - } - - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - return response - -openconfig-network-instance:network-instance \ No newline at end of file + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + # Arista Parsing Fixes: + # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 + # overwrite with .../hellos/config/hello-interval + network_instances = json_data.get('openconfig-network-instance:network-instance', []) + for network_instance in network_instances: + if network_instance['name'] != 'default': continue + mpls_rsvp_te = network_instance.get('mpls', {}).get('signaling-protocols', {}).get('rsvp-te', {}) + mpls_rsvp_te_hellos = mpls_rsvp_te.get('global', {}).get('hellos', {}) + hello_interval = mpls_rsvp_te_hellos.get('config', {}).get('hello-interval', 9000) + mpls_rsvp_te_hellos.get('state', {})['hello-interval'] = hello_interval + + yang_network_instances_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_network_instances_path, json_data, fmt='json', strict=False) + + entries = [] + for network_instance in json_data_valid['network-instances']['network-instance']: + LOGGER.debug('network_instance={:s}'.format(str(network_instance))) + ni_name = network_instance['name'] + + ni_config = network_instance['config'] + ni_type = ni_config['type'].split(':')[-1] + + _net_inst = {'name': ni_name, 'type': ni_type} + entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) + entries.append((entry_net_inst_key, _net_inst)) + + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) + for ni_protocol in ni_protocols: + ni_protocol_id = ni_protocol['identifier'].split(':')[-1] + ni_protocol_name = ni_protocol['name'] + + _protocol = {'id': ni_protocol_id, 'name': ni_protocol_name} + entry_protocol_key = '{:s}/protocol[{:s}]'.format(entry_net_inst_key, ni_protocol_id) + entries.append((entry_protocol_key, _protocol)) + + if ni_protocol_id == 'STATIC': + static_routes = ni_protocol.get('static-routes', {}).get('static', []) + for static_route in static_routes: + static_route_prefix = static_route['prefix'] + + next_hops = static_route.get('next-hops', {}).get('next-hop', []) + _next_hops = [ + { + 'index' : next_hop['index'], + 'gateway': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], + } + for next_hop in next_hops + ] + _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + + _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + entry_static_route_key = '{:s}/static_routes[{:s}]'.format( + entry_protocol_key, static_route_prefix + ) + entries.append((entry_static_route_key, _static_route)) + + ni_tables = network_instance.get('tables', {}).get('table', []) + for ni_table in ni_tables: + ni_table_protocol = ni_table['protocol'].split(':')[-1] + ni_table_address_family = ni_table['address-family'].split(':')[-1] + _table = {'protocol': ni_table_protocol, 'address_family': ni_table_address_family} + entry_table_key = '{:s}/table[{:s},{:s}]'.format( + entry_net_inst_key, ni_table_protocol, ni_table_address_family + ) + entries.append((entry_table_key, _table)) + + ni_vlans = network_instance.get('vlans', {}).get('vlan', []) + for ni_vlan in ni_vlans: + ni_vlan_id = ni_vlan['vlan-id'] + + #ni_vlan_config = ni_vlan['config'] + ni_vlan_state = ni_vlan['state'] + ni_vlan_name = ni_vlan_state['name'] + + _members = [ + member['state']['interface'] + for member in ni_vlan.get('members', {}).get('member', []) + ] + _vlan = {'vlan_id': ni_vlan_id, 'name': ni_vlan_name, 'members': _members} + entry_vlan_key = '{:s}/vlan[{:d}]'.format(entry_net_inst_key, ni_vlan_id) + entries.append((entry_vlan_key, _vlan)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 8cf704e29..dfb8eabaf 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable, Optional +from typing import Any, Callable, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -40,3 +40,22 @@ def container_get_first( if namespace_key_name in container: return container[namespace_key_name] return default + +def get_value( + resource_value : Dict, field_name : str, cast_func : Callable = lambda x:x, default : Optional[Any] = None +) -> Optional[Any]: + field_value = resource_value.get(field_name, default) + if field_value is not None: field_value = cast_func(field_value) + return field_value + +def get_bool(resource_value : Dict, field_name : bool, default : Optional[Any] = None) -> bool: + return get_value(resource_value, field_name, cast_func=bool, default=default) + +def get_float(resource_value : Dict, field_name : float, default : Optional[Any] = None) -> float: + return get_value(resource_value, field_name, cast_func=float, default=default) + +def get_int(resource_value : Dict, field_name : int, default : Optional[Any] = None) -> int: + return get_value(resource_value, field_name, cast_func=int, default=default) + +def get_str(resource_value : Dict, field_name : str, default : Optional[Any] = None) -> str: + return get_value(resource_value, field_name, cast_func=str, default=default) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py new file mode 100644 index 000000000..fe8672187 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py @@ -0,0 +1,109 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, libyang, logging, os +from typing import Dict, Optional + +YANG_BASE_PATH = os.path.join(os.path.dirname(__file__), '..', 'git', 'openconfig', 'public') +YANG_SEARCH_PATHS = ':'.join([ + os.path.join(YANG_BASE_PATH, 'release'), + os.path.join(YANG_BASE_PATH, 'third_party'), +]) + +YANG_MODULES = [ + 'iana-if-type', + 'openconfig-vlan-types', + + 'openconfig-interfaces', + 'openconfig-if-8021x', + 'openconfig-if-aggregate', + 'openconfig-if-ethernet-ext', + 'openconfig-if-ethernet', + 'openconfig-if-ip-ext', + 'openconfig-if-ip', + 'openconfig-if-poe', + 'openconfig-if-sdn-ext', + 'openconfig-if-tunnel', + + 'openconfig-vlan', + + 'openconfig-types', + 'openconfig-policy-types', + 'openconfig-mpls-types', + 'openconfig-network-instance-types', + 'openconfig-network-instance', + + 'openconfig-platform', + 'openconfig-platform-controller-card', + 'openconfig-platform-cpu', + 'openconfig-platform-ext', + 'openconfig-platform-fabric', + 'openconfig-platform-fan', + 'openconfig-platform-integrated-circuit', + 'openconfig-platform-linecard', + 'openconfig-platform-pipeline-counters', + 'openconfig-platform-port', + 'openconfig-platform-psu', + 'openconfig-platform-software', + 'openconfig-platform-transceiver', + 'openconfig-platform-types', +] + +LOGGER = logging.getLogger(__name__) + +class YangHandler: + def __init__(self) -> None: + self._yang_context = libyang.Context(YANG_SEARCH_PATHS) + self._loaded_modules = set() + for yang_module_name in YANG_MODULES: + LOGGER.info('Loading module: {:s}'.format(str(yang_module_name))) + self._yang_context.load_module(yang_module_name).feature_enable_all() + self._loaded_modules.add(yang_module_name) + self._data_path_instances = dict() + + def get_data_paths(self) -> Dict[str, libyang.DNode]: + return self._data_path_instances + + def get_data_path(self, path : str) -> libyang.DNode: + data_path_instance = self._data_path_instances.get(path) + if data_path_instance is None: + data_path_instance = self._yang_context.create_data_path(path) + self._data_path_instances[path] = data_path_instance + return data_path_instance + + def parse_to_dict( + self, request_path : str, json_data : Dict, fmt : str = 'json', strict : bool = True + ) -> Dict: + if fmt != 'json': raise Exception('Unsupported format: {:s}'.format(str(fmt))) + LOGGER.debug('request_path = {:s}'.format(str(request_path))) + LOGGER.debug('json_data = {:s}'.format(str(json_data))) + LOGGER.debug('format = {:s}'.format(str(fmt))) + + parent_path_parts = list(filter(lambda s: len(s) > 0, request_path.split('/'))) + for parent_path_part in reversed(parent_path_parts): + json_data = {parent_path_part: json_data} + str_data = json.dumps(json_data) + + dnode : Optional[libyang.DNode] = self._yang_context.parse_data_mem( + str_data, fmt, strict=strict, parse_only=True, #validate_present=True, #validate=True, + ) + if dnode is None: raise Exception('Unable to parse Data({:s})'.format(str(json_data))) + + parsed = dnode.print_dict() + LOGGER.debug('parsed = {:s}'.format(json.dumps(parsed))) + dnode.free() + return parsed + + def destroy(self) -> None: + self._yang_context.destroy() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py index d20c77b11..a03692d95 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Any, Dict, List, Tuple +from .YangHandler import YangHandler class _Handler: def get_resource_key(self) -> str: @@ -23,10 +24,14 @@ class _Handler: # Retrieve the OpenConfig path schema used to interrogate the device raise NotImplementedError() - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: # Compose a Set/Delete message based on the resource_key/resource_value fields, and the delete flag raise NotImplementedError() - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: # Parse a Reply from the device and return a list of resource_key/resource_value pairs raise NotImplementedError() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 6d54ef28d..38bc4db40 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES from ._Handler import _Handler from .Component import ComponentHandler @@ -23,6 +23,7 @@ from .NetworkInstance import NetworkInstanceHandler from .NetworkInstanceInterface import NetworkInstanceInterfaceHandler from .NetworkInstanceStaticRoute import NetworkInstanceStaticRouteHandler from .Tools import get_schema +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -71,7 +72,8 @@ PATH_TO_HANDLER = { } def get_handler( - resource_key : Optional[str] = None, path : Optional[str] = None, raise_if_not_found=True + resource_key : Optional[str] = None, path : Optional[str] = None, + raise_if_not_found=True ) -> Optional[_Handler]: if (resource_key is None) == (path is None): MSG = 'Exactly one of resource_key({:s}) or path({:s}) must be specified' @@ -95,10 +97,18 @@ def get_handler( return handler def get_path(resource_key : str) -> str: - return get_handler(resource_key=resource_key).get_path() + handler = get_handler(resource_key=resource_key) + return handler.get_path() -def parse(str_path : str, value : Union[Dict, List]): - return get_handler(path=str_path).parse(value) +def parse( + str_path : str, value : Union[Dict, List], yang_handler : YangHandler +) -> List[Tuple[str, Dict[str, Any]]]: + handler = get_handler(path=str_path) + return handler.parse(value, yang_handler) -def compose(resource_key : str, resource_value : Union[Dict, List], delete : bool = False) -> Tuple[str, str]: - return get_handler(resource_key=resource_key).compose(resource_key, resource_value, delete=delete) +def compose( + resource_key : str, resource_value : Union[Dict, List], + yang_handler : YangHandler, delete : bool = False +) -> Tuple[str, str]: + handler = get_handler(resource_key=resource_key) + return handler.compose(resource_key, resource_value, yang_handler, delete=delete) diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py index b90bf3db8..4c202da2c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py @@ -17,7 +17,7 @@ from common.tools.grpc.Tools import grpc_message_to_json from ..gnmi.gnmi_pb2 import CapabilityRequest # pylint: disable=no-name-in-module from ..gnmi.gnmi_pb2_grpc import gNMIStub -def get_supported_encodings( +def check_capabilities( stub : gNMIStub, username : str, password : str, timeout : Optional[int] = None ) -> Set[Union[str, int]]: metadata = [('username', username), ('password', password)] @@ -25,6 +25,17 @@ def get_supported_encodings( reply = stub.Capabilities(req, metadata=metadata, timeout=timeout) data = grpc_message_to_json(reply) + + gnmi_version = data.get('gNMI_version') + if gnmi_version is None or gnmi_version != '0.7.0': + raise Exception('Unsupported gNMI version: {:s}'.format(str(gnmi_version))) + + #supported_models = { + # supported_model['name']: supported_model['version'] + # for supported_model in data.get('supported_models', []) + #} + # TODO: check supported models and versions + supported_encodings = { supported_encoding for supported_encoding in data.get('supported_encodings', []) @@ -33,4 +44,6 @@ def get_supported_encodings( if len(supported_encodings) == 0: # pylint: disable=broad-exception-raised raise Exception('No supported encodings found') - return supported_encodings + if 'JSON_IETF' not in supported_encodings: + # pylint: disable=broad-exception-raised + raise Exception('JSON_IETF encoding not supported') diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 9933cb858..73e43b87c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -61,7 +61,7 @@ def decode_value(value : TypedValue) -> Any: str_value : str = value.json_ietf_val.decode('UTF-8') try: # Cleanup and normalize the records according to OpenConfig - str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + #str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') json_value = json.loads(str_value) recursive_remove_keys(json_value) return json_value diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py deleted file mode 100644 index 684b9f4c3..000000000 --- a/src/device/tests/test_gnmi.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, os, sys, time -from typing import Dict, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | # | Name | Container ID | Image | Kind | State | IPv4 Address | IPv6 Address | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | 1 | clab-tfs-scenario-client1 | a8d48ec3265a | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.201/24 | N/A | -# | 2 | clab-tfs-scenario-client2 | fc88436d2b32 | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.202/24 | N/A | -# | 3 | clab-tfs-scenario-srl1 | b995b9bdadda | ghcr.io/nokia/srlinux | srl | running | 172.100.100.101/24 | N/A | -# | 4 | clab-tfs-scenario-srl2 | aacfc38cc376 | ghcr.io/nokia/srlinux | srl | running | 172.100.100.102/24 | N/A | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = {'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix} - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def main(): - driver_settings = { - 'protocol': 'gnmi', - 'username': 'admin', - 'password': 'admin', - 'use_tls' : False, - } - driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) - driver.Connect() - - #resources_to_get = [] - #resources_to_get = [RESOURCE_ENDPOINTS] - #resources_to_get = [RESOURCE_INTERFACES] - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - #resources_to_get = [RESOURCE_ROUTING_POLICIES] - #resources_to_get = [RESOURCE_SERVICES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #resources_to_set = [ - # network_instance('test-svc', 'L3VRF'), - # - # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/1', 0), - # - # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/2', 0), - # - # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), - # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), - #] - #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - #results_setconfig = driver.SetConfig(resources_to_set) - #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - #resources_to_delete = [ - # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - # - # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - # - # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - # - # #network_instance('20f66fb5', 'L3VRF'), - #] - #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - #results_deleteconfig = driver.DeleteConfig(resources_to_delete) - #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - time.sleep(1) - - driver.Disconnect() - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..4c2dca5d5 --- /dev/null +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -0,0 +1,616 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict, List, Tuple +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### STORAGE POPULATORS ############################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = resource_value.get('members') + continue + + +##### EXPECTED CONFIG COMPOSERS ######################################################################################## + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config + + +##### REQUEST COMPOSERS ################################################################################################ + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = {'name': ni_name, 'type': ni_type} + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} + return str_path, str_data + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[test-l3-svc]', { + 'name': 'test-l3-svc', 'type': 'L3VRF' + }), + ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +#def test_unitary_gnmi_openconfig( +# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +#) -> None: +# #resources_to_get = [] +# resources_to_get = [RESOURCE_ENDPOINTS] +# #resources_to_get = [RESOURCE_INTERFACES] +# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] +# #resources_to_get = [RESOURCE_ROUTING_POLICIES] +# #resources_to_get = [RESOURCE_SERVICES] +# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) +# results_getconfig = driver.GetConfig(resources_to_get) +# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) +# +# #resources_to_set = [ +# # network_instance('test-svc', 'L3VRF'), +# # +# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/1', 0), +# # +# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/2', 0), +# # +# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), +# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), +# #] +# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) +# #results_setconfig = driver.SetConfig(resources_to_set) +# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) +# +# #resources_to_delete = [ +# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), +# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), +# # +# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), +# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), +# # +# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), +# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), +# # +# # #network_instance('20f66fb5', 'L3VRF'), +# #] +# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) +# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) +# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From f49ad8703b3e2cd54614f4ac95d22689e9ea67ba Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:40:02 +0000 Subject: [PATCH 04/34] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Corrected basic unitary test for network instances --- .../tests/test_unitary_gnmi_openconfig.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py index 4c2dca5d5..7d33d1a71 100644 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -141,7 +141,7 @@ def populate_network_instances_storage( prefix = match.group(3) ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) continue match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) @@ -161,7 +161,7 @@ def populate_network_instances_storage( ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) ni_v_storage['vlan_id'] = vlan_id ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = resource_value.get('members') + ni_v_storage['members'] = sorted(resource_value.get('members')) continue @@ -389,6 +389,12 @@ def test_get_network_instances( populate_network_instances_storage(storage, results_getconfig) expected_getconfig = get_expected_network_instance_config(storage) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) @@ -481,6 +487,9 @@ def test_set_network_instances( 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' }) ]) + for resource_key, resource_value in expected_getconfig: + if resource_key == '/network_instance[default]/vlan[1]': + resource_value['members'] = list() LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) permitted_retries = 5 @@ -490,6 +499,12 @@ def test_set_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs == 0: break @@ -562,6 +577,12 @@ def test_del_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + expected_getconfig = get_expected_network_instance_config(storage) diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) -- GitLab From a3df08b8600e8aa1c54e27516b5784a126f881c2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:57:04 +0000 Subject: [PATCH 05/34] DataPlane-in-a-box: - Added scripts to connect to cEOS CLI - Updated cEOS version in ContainerLab descriptors --- dataplane-in-a-box/arista.clab.yml | 3 ++- dataplane-in-a-box/ceos-cli-wan1.sh | 3 +++ dataplane-in-a-box/ceos-cli-wan2.sh | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100755 dataplane-in-a-box/ceos-cli-wan1.sh create mode 100755 dataplane-in-a-box/ceos-cli-wan2.sh diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 9a8bff73f..4f3b77129 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -24,7 +24,8 @@ topology: kinds: arista_ceos: kind: arista_ceos - image: ceos:4.30.4M + #image: ceos:4.30.4M + image: ceos:4.31.1F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/ceos-cli-wan1.sh new file mode 100755 index 000000000..4ae21bcb5 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan1 Cli diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/ceos-cli-wan2.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli -- GitLab From 32d82663c45cb5a53f2980e066a838cc571b6b70 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:58:30 +0000 Subject: [PATCH 06/34] Device component - GNMI OpenConfig: - advanced development of driver - updated unitary tests and related scripts --- ...un_tests_locally-device-gnmi-openconfig.sh | 2 +- .../handlers/InterfaceCounter.py | 66 +- .../handlers/NetworkInstance.py | 34 +- .../handlers/NetworkInstanceInterface.py | 52 +- .../handlers/NetworkInstanceStaticRoute.py | 79 ++- src/device/tests/gnmi_openconfig/__init__.py | 14 + .../gnmi_openconfig/request_composers.py | 44 ++ src/device/tests/gnmi_openconfig/storage.py | 285 ++++++++ .../test_unitary_gnmi_openconfig.py | 556 +++++++++++++++ .../tests/test_unitary_gnmi_openconfig.py | 637 ------------------ 10 files changed, 1051 insertions(+), 718 deletions(-) create mode 100644 src/device/tests/gnmi_openconfig/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/request_composers.py create mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py delete mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh index d81684da1..7183b4104 100755 --- a/scripts/run_tests_locally-device-gnmi-openconfig.sh +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -22,4 +22,4 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc # Run unitary tests and analyze coverage of code at same time # helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ - device/tests/test_unitary_gnmi_openconfig.py + device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index 1c2cfc17a..d4701826c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple -import pyangbind.lib.pybindJSON as pybindJSON -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -25,40 +24,41 @@ class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - counters = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - )) + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') - interface = {} - interface['name'] = oc_interface.name - - interface_counters = oc_interface.state.counters - interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts - interface['in-discards' ] = interface_counters.in_discards - interface['in-errors' ] = interface_counters.in_errors - interface['in-fcs-errors' ] = interface_counters.in_fcs_errors - interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts - interface['in-octets' ] = interface_counters.in_octets - interface['in-pkts' ] = interface_counters.in_pkts - interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts - interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts - interface['out-discards' ] = interface_counters.out_discards - interface['out-errors' ] = interface_counters.out_errors - interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts - interface['out-octets' ] = interface_counters.out_octets - interface['out-pkts' ] = interface_counters.out_pkts - interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + entries = [] + for interface in json_data_valid['interfaces']['interface']: + LOGGER.info('interface={:s}'.format(str(interface))) + interface_name = interface['name'] + interface_counters = interface.get('state', {}).get('counters', {}) + _interface = { + 'name' : interface_name, + 'in-broadcast-pkts' : interface_counters['in_broadcast_pkts' ], + 'in-discards' : interface_counters['in_discards' ], + 'in-errors' : interface_counters['in_errors' ], + 'in-fcs-errors' : interface_counters['in_fcs_errors' ], + 'in-multicast-pkts' : interface_counters['in_multicast_pkts' ], + 'in-octets' : interface_counters['in_octets' ], + 'in-pkts' : interface_counters['in_pkts' ], + 'in-unicast-pkts' : interface_counters['in_unicast_pkts' ], + 'out-broadcast-pkts': interface_counters['out_broadcast_pkts'], + 'out-discards' : interface_counters['out_discards' ], + 'out-errors' : interface_counters['out_errors' ], + 'out-multicast-pkts': interface_counters['out_multicast_pkts'], + 'out-octets' : interface_counters['out_octets' ], + 'out-pkts' : interface_counters['out_pkts' ], + 'out-unicast-pkts' : interface_counters['out_unicast_pkts' ], + } LOGGER.info('interface = {:s}'.format(str(interface))) - if len(interface) == 0: continue - counters.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) - return counters + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 0b4d15745..1efed024c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, libyang, logging -import operator +import json, libyang, logging, operator from typing import Any, Dict, List, Tuple from ._Handler import _Handler -from .Tools import get_bool, get_int, get_str +from .Tools import get_str from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -45,7 +44,7 @@ class NetworkInstanceHandler(_Handler): def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - ni_name = get_str(resource_value, 'name') # test-svc + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -56,21 +55,32 @@ class NetworkInstanceHandler(_Handler): ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # 'DIRECTLY_CONNECTED' is implicitly added + str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) + #str_data = json.dumps({ + # 'name': ni_name, + # 'config': {'name': ni_name, 'type': ni_type}, + #}) + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni_path = 'network-instance[name="{:s}"]'.format(ni_name) + yang_ni : libyang.DContainer = yang_nis.create_path(yang_ni_path) + yang_ni.create_path('config/name', ni_name) + yang_ni.create_path('config/type', ni_type) - str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) - str_data = json.dumps({ - 'name': ni_name, - 'config': {'name': ni_name, 'type': ni_type}, - #'protocols': {'protocol': protocols}, - }) + # 'DIRECTLY_CONNECTED' is implicitly added + #'protocols': {'protocol': protocols}, + + str_data = yang_ni.print_mem('json') + LOGGER.warning('str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:network-instance'][0] + str_data = json.dumps(json_data) return str_path, str_data def parse( self, json_data : Dict, yang_handler : YangHandler ) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) # Arista Parsing Fixes: # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index 205373fca..ab105c2b0 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -12,21 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +IS_CEOS = True + class NetworkInstanceInterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/interface' - def get_path(self) -> str: return '/network-instances/network-instance/interfaces' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - if_name = str(resource_value['if_name' ]) # ethernet-1/1 - sif_index = int(resource_value['sif_index']) # 0 - if_id = '{:s}.{:d}'.format(if_name, sif_index) + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sif_index', 0) # 0 + + if IS_CEOS: + if_id = if_name + else: + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -35,12 +45,30 @@ class NetworkInstanceInterfaceHandler(_Handler): return str_path, str_data str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) - str_data = json.dumps({ - 'id': if_id, - 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, - }) + #str_data = json.dumps({ + # 'id': if_id, + # 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_ifs : libyang.DContainer = yang_ni.create_path('interfaces') + yang_ni_if_path = 'interface[id="{:s}"]'.format(if_id) + yang_ni_if : libyang.DContainer = yang_ni_ifs.create_path(yang_ni_if_path) + yang_ni_if.create_path('config/id', if_id) + yang_ni_if.create_path('config/interface', if_name) + yang_ni_if.create_path('config/subinterface', sif_index) + + str_data = yang_ni_if.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 9d75e9ac6..0343e3cba 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -12,21 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) class NetworkInstanceStaticRouteHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/static_route' - def get_path(self) -> str: return '/network-instances/network-instance/static_route' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/static_route' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - prefix = str(resource_value['prefix' ]) # '172.0.1.0/24' + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + prefix = get_str(resource_value, 'prefix') # '172.0.1.0/24' - identifier = 'STATIC' + identifier = 'openconfig-policy-types:STATIC' name = 'static' if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols' @@ -35,27 +39,56 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = str(resource_value['next_hop' ]) # '172.0.0.1' - next_hop_index = int(resource_value.get('next_hop_index', 0)) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) - str_data = json.dumps({ - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': True}, - 'static_routes': {'static': [{ - 'prefix': prefix, - 'config': {'prefix': prefix}, - 'next_hops': { - 'next-hop': [{ - 'index': next_hop_index, - 'config': {'index': next_hop_index, 'next_hop': next_hop} - }] - } - }]} - }) + #str_data = json.dumps({ + # 'identifier': identifier, 'name': name, + # 'config': {'identifier': identifier, 'name': name, 'enabled': True}, + # 'static_routes': {'static': [{ + # 'prefix': prefix, + # 'config': {'prefix': prefix}, + # 'next_hops': { + # 'next-hop': [{ + # 'index': next_hop_index, + # 'config': {'index': next_hop_index, 'next_hop': next_hop} + # }] + # } + # }]} + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_prs : libyang.DContainer = yang_ni.create_path('protocols') + yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, name) + yang_ni_pr : libyang.DContainer = yang_ni_prs.create_path(yang_ni_pr_path) + yang_ni_pr.create_path('config/identifier', identifier) + yang_ni_pr.create_path('config/name', name ) + yang_ni_pr.create_path('config/enabled', True ) + + yang_ni_pr_srs : libyang.DContainer = yang_ni_pr.create_path('static-routes') + yang_ni_pr_sr_path = 'static[prefix="{:s}"]'.format(prefix) + yang_ni_pr_sr : libyang.DContainer = yang_ni_pr_srs.create_path(yang_ni_pr_sr_path) + yang_ni_pr_sr.create_path('config/prefix', prefix) + + yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') + yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) + yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) + yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) + + str_data = yang_ni_pr.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:protocol'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/tests/gnmi_openconfig/__init__.py b/src/device/tests/gnmi_openconfig/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/request_composers.py new file mode 100644 index 000000000..faa8425c8 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/request_composers.py @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = { + 'name': ni_name, 'type': ni_type + } + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = { + 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + } + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = { + 'name': ni_name, 'if_name': if_name, 'sif_index': sif_index + } + return str_path, str_data diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py new file mode 100644 index 000000000..4271b002f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage.py @@ -0,0 +1,285 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest, re +from typing import Dict, List, Tuple + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### POPULATE INTERFACE STORAGE ####################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + + +##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = sorted(resource_value.get('members')) + continue + + +##### GET EXPECTED INTERFACE CONFIG #################################################################################### + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + + +##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..69b7a609a --- /dev/null +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -0,0 +1,556 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route +from .storage import ( # pylint: disable=unused-import + storage, # be careful, order of symbols is important here!; storage should be the first one + get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, + populate_network_instances_storage +) + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + + +##### DRIVER FIXTURE ################################################################################################### + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + + +##### NETWORK INSTANCE DETAILS ######################################################################################### + +NI_NAME = 'test-l3-svc' +NI_TYPE = 'L3VRF' +NI_INTERFACES = [ + # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled + ('Ethernet1', 0, '192.168.1.1', 24, True), + ('Ethernet10', 0, '192.168.10.1', 24, True), +] +NI_STATIC_ROUTES = [ + # prefix, gateway, metric + ('172.0.0.0/24', '172.16.0.2', 1), + ('172.2.0.0/24', '172.16.0.3', 1), +] + + +##### TEST METHODS ##################################################################################################### + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance(NI_NAME, NI_TYPE), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + #for resource_key, resource_value in expected_getconfig: + # if resource_key == '/network_instance[default]/vlan[1]': + # resource_value['members'] = list() + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) + for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted([ + if_name + for if_name, _, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_add_interfaces_to_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_set_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) + for prefix, gateway, metric in NI_STATIC_ROUTES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + + + + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + # + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), + network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_deleteconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + #expected_getconfig = get_expected_interface_config(storage) + + #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + #num_diffs = len(diff_data) + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces_from_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interface_ids = sorted([ + '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ]) + results = set(results_deleteconfig) + assert len(results) == len(interface_ids) + for interface_id in interface_ids: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance(NI_NAME, 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py deleted file mode 100644 index 7d33d1a71..000000000 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ /dev/null @@ -1,637 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import deepdiff, logging, os, pytest, re, time -from typing import Dict, List, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' - -# pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -DRIVER_SETTING_ADDRESS = '172.20.20.101' -DRIVER_SETTING_PORT = 6030 -DRIVER_SETTING_USERNAME = 'admin' -DRIVER_SETTING_PASSWORD = 'admin' -DRIVER_SETTING_USE_TLS = False - -@pytest.fixture(scope='session') -def driver() -> GnmiOpenConfigDriver: - _driver = GnmiOpenConfigDriver( - DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, - username=DRIVER_SETTING_USERNAME, - password=DRIVER_SETTING_PASSWORD, - use_tls=DRIVER_SETTING_USE_TLS, - ) - _driver.Connect() - yield _driver - time.sleep(1) - _driver.Disconnect() - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### STORAGE POPULATORS ############################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### EXPECTED CONFIG COMPOSERS ######################################################################################## - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config - - -##### REQUEST COMPOSERS ################################################################################################ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = { - 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix - } - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def test_get_endpoints( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[test-l3-svc]', { - 'name': 'test-l3-svc', 'type': 'L3VRF' - }), - ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - for resource_key, resource_value in expected_getconfig: - if resource_key == '/network_instance[default]/vlan[1]': - resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - -#def test_unitary_gnmi_openconfig( -# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -#) -> None: -# #resources_to_get = [] -# resources_to_get = [RESOURCE_ENDPOINTS] -# #resources_to_get = [RESOURCE_INTERFACES] -# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] -# #resources_to_get = [RESOURCE_ROUTING_POLICIES] -# #resources_to_get = [RESOURCE_SERVICES] -# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) -# results_getconfig = driver.GetConfig(resources_to_get) -# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) -# -# #resources_to_set = [ -# # network_instance('test-svc', 'L3VRF'), -# # -# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/1', 0), -# # -# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/2', 0), -# # -# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), -# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), -# #] -# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) -# #results_setconfig = driver.SetConfig(resources_to_set) -# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) -# -# #resources_to_delete = [ -# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), -# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), -# # -# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), -# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), -# # -# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), -# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), -# # -# # #network_instance('20f66fb5', 'L3VRF'), -# #] -# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) -# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) -# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From 74a75434c591e0a4a74d94c64ea61600b2cdc3e2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 17:04:35 +0000 Subject: [PATCH 07/34] Device component - GNMI OpenConfig: - Corrected management of network instance interfaces - Improved unitary tests --- .../handlers/NetworkInstance.py | 12 ++ .../handlers/NetworkInstanceInterface.py | 8 +- .../test_unitary_gnmi_openconfig.py | 178 +++++++++++------- 3 files changed, 125 insertions(+), 73 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 1efed024c..b97612987 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -108,6 +108,18 @@ class NetworkInstanceHandler(_Handler): entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) entries.append((entry_net_inst_key, _net_inst)) + ni_interfaces = network_instance.get('interfaces', {}).get('interface', []) + for ni_interface in ni_interfaces: + #ni_if_id = ni_interface['id'] + ni_if_config = ni_interface['config'] + ni_if_name = ni_if_config['interface'] + ni_sif_index = ni_if_config['subinterface'] + ni_if_id = '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + + _interface = {'name': ni_name, 'id': ni_if_id, 'if_name': ni_if_name, 'sif_index': ni_sif_index} + entry_interface_key = '{:s}/interface[{:s}]'.format(entry_net_inst_key, ni_if_id) + entries.append((entry_interface_key, _interface)) + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) for ni_protocol in ni_protocols: ni_protocol_id = ni_protocol['identifier'].split(':')[-1] diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index ab105c2b0..af2178fe9 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -32,11 +32,7 @@ class NetworkInstanceInterfaceHandler(_Handler): ni_name = get_str(resource_value, 'name' ) # test-svc if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 sif_index = get_int(resource_value, 'sif_index', 0) # 0 - - if IS_CEOS: - if_id = if_name - else: - if_id = '{:s}.{:d}'.format(if_name, sif_index) + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -44,6 +40,8 @@ class NetworkInstanceInterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data + if IS_CEOS: if_id = if_name + str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) #str_data = json.dumps({ # 'id': if_id, diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 69b7a609a..47c8e1cdb 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -141,7 +141,6 @@ def test_get_network_instances( num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 - raise Exception() def test_set_network_instances( @@ -284,39 +283,80 @@ def test_add_interfaces_to_network_instance( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + interfaces = sorted([ + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_set_network_instance_static_routes( @@ -336,43 +376,45 @@ def test_set_network_instance_static_routes( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + prefixes = sorted([ + prefix + for prefix, _, _ in NI_STATIC_ROUTES + ]) + results = set(results_setconfig) + assert len(results) == len(prefixes) + for prefix in prefixes: + assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { + 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric + }) + for prefix, gateway, metric in NI_STATIC_ROUTES + ]) + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 - # - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_del_network_instance_static_routes( @@ -429,16 +471,16 @@ def test_del_interfaces_from_network_instance( LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) resources_to_delete = [ - network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) interface_ids = sorted([ - '{:s}.{:d}'.format(ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ]) results = set(results_deleteconfig) assert len(results) == len(interface_ids) -- GitLab From 03f748da04512622f43bb3c0551737434a25ec72 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 10 Jan 2024 18:36:50 +0000 Subject: [PATCH 08/34] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/storage.py | 285 -------- .../tests/gnmi_openconfig/storage/Storage.py | 23 + .../storage/StorageEndpoints.py | 72 ++ .../storage/StorageInterface.py | 122 ++++ .../storage/StorageNetworkInstance.py | 194 ++++++ .../tests/gnmi_openconfig/storage/Tools.py | 32 + .../tests/gnmi_openconfig/storage/__init__.py | 14 + .../test_unitary_gnmi_openconfig.py | 656 ++++++------------ .../tests/gnmi_openconfig/tools/__init__.py | 14 + .../gnmi_openconfig/tools/check_config.py | 82 +++ .../gnmi_openconfig/tools/check_updates.py | 21 + .../tools/expected_config_composers.py | 58 ++ .../{ => tools}/request_composers.py | 0 .../tools/result_config_adapters.py | 29 + 14 files changed, 864 insertions(+), 738 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageInterface.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Tools.py create mode 100644 src/device/tests/gnmi_openconfig/storage/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_config.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_updates.py create mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py rename src/device/tests/gnmi_openconfig/{ => tools}/request_composers.py (100%) create mode 100644 src/device/tests/gnmi_openconfig/tools/result_config_adapters.py diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py deleted file mode 100644 index 4271b002f..000000000 --- a/src/device/tests/gnmi_openconfig/storage.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest, re -from typing import Dict, List, Tuple - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### POPULATE INTERFACE STORAGE ####################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - - -##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### GET EXPECTED INTERFACE CONFIG #################################################################################### - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - - -##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/storage/Storage.py b/src/device/tests/gnmi_openconfig/storage/Storage.py new file mode 100644 index 000000000..4aaf29c99 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Storage.py @@ -0,0 +1,23 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .StorageEndpoints import StorageEndpoints +from .StorageInterface import StorageInterface +from .StorageNetworkInstance import StorageNetworkInstance + +class Storage: + def __init__(self) -> None: + self.endpoints = StorageEndpoints() + self.interfaces = StorageInterface() + self.network_instances = StorageNetworkInstance() diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py new file mode 100644 index 000000000..815a1b0ad --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -0,0 +1,72 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +RE_RESKEY_ENDPOINT = re.compile(r'^\/endpoints\/endpoint\[([^\]]+)\]$') + +ENDPOINT_PACKET_SAMPLE_TYPES : Dict[int, str] = { + 101: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-pkts', + 102: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-pkts', + 201: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-octets', + 202: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-octets', +} + +class Endpoints: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/endpoints/endpoint[{:s}]', ['uuid', 'type', 'sample_types']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ep_uuid : str, resource_value : Dict) -> None: + item = self._items.setdefault(ep_uuid, dict()) + item['uuid'] = ep_uuid + + for _, field_names in Endpoints.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + item['sample_types'] = { + sample_type_id : sample_type_path.format(ep_uuid) + for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() + } + + def remove(self, ep_uuid : str) -> None: + self._items.pop(ep_uuid, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Endpoints.STRUCT) + +class StorageEndpoints: + def __init__(self) -> None: + self.endpoints = Endpoints() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_ENDPOINT.match(resource_key) + if match is not None: + self.endpoints.add(match.group(1), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.endpoints.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py new file mode 100644 index 000000000..a0391e92f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/interface\[([^\]]+)\]' +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'$') +RE_RESKEY_ETHERNET = re.compile(PREFIX + r'\/ethernet$') +RE_RESKEY_SUBINTERFACE = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]$') +RE_RESKEY_IPV4_ADDRESS = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$') + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]', ['name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', + 'hardware-port', 'transceiver']), + ('/interface[{:s}]/ethernet', ['port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, if_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(if_name, dict()) + item['name'] = if_name + for _, field_names in Interfaces.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + def remove(self, if_name : str) -> None: + self._items.pop(if_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class SubInterfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]', ['index']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, if_name : str, subif_index : int) -> None: + item = self._items.setdefault((if_name, subif_index), dict()) + item['index'] = subif_index + + def remove(self, if_name : str, subif_index : int) -> None: + self._items.pop((if_name, subif_index), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, SubInterfaces.STRUCT) + +class IPv4Addresses: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]', ['ip', 'origin', 'prefix']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int, str], Dict] = dict() + + def add(self, if_name : str, subif_index : int, ipv4_address : str, resource_value : Dict) -> None: + item = self._items.setdefault((if_name, subif_index, ipv4_address), dict()) + item['ip' ] = ipv4_address + item['origin'] = resource_value.get('origin') + item['prefix'] = resource_value.get('prefix') + + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: + self._items.pop((if_name, subif_index, ipv4_address), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, IPv4Addresses.STRUCT) + +class StorageInterface: + def __init__(self) -> None: + self.interfaces = Interfaces() + self.subinterfaces = SubInterfaces() + self.ipv4_addresses = IPv4Addresses() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_ETHERNET.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_SUBINTERFACE.match(resource_key) + if match is not None: + self.subinterfaces.add(match.group(1), int(match.group(2))) + continue + + match = RE_RESKEY_IPV4_ADDRESS.match(resource_key) + if match is not None: + self.ipv4_addresses.add(match.group(1), int(match.group(2)), match.group(3), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.subinterfaces.compose_resources()) + expected_config.extend(self.ipv4_addresses.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py new file mode 100644 index 000000000..558cc032c --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -0,0 +1,194 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/network\_instance\[([^\]]+)\]' +RE_RESKEY_NET_INST = re.compile(PREFIX + r'$') +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'\/interface\[([^\]]+)\]$') +RE_RESKEY_PROTOCOL = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]$') +RE_RESKEY_PROTO_STATIC = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$') +RE_RESKEY_TABLE = re.compile(PREFIX + r'\/table\[([^\,]+)\,([^\]]+)\]$') +RE_RESKEY_VLAN = re.compile(PREFIX + r'\/vlan\[([^\]]+)\]$') + +class NetworkInstances: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]', ['name', 'type']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ni_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(ni_name, dict()) + item['name'] = ni_name + item['type'] = resource_value.get('type') + + def remove(self, ni_name : str) -> None: + self._items.pop(ni_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, NetworkInstances.STRUCT) + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, if_name : str) -> None: + item = self._items.setdefault((ni_name, if_name), dict()) + item['ni_name'] = ni_name + item['if_name'] = if_name + + def remove(self, ni_name : str, if_name : str) -> None: + self._items.pop((ni_name, if_name), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class Protocols: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]', ['id', 'name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str) -> None: + item = self._items.setdefault((ni_name, protocol), dict()) + item['id' ] = protocol + item['name'] = protocol + + def remove(self, ni_name : str, protocol : str) -> None: + self._items.pop((ni_name, protocol), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Protocols.STRUCT) + +class StaticRoutes: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, protocol, prefix), dict()) + item['prefix' ] = prefix + item['next_hops'] = sorted(resource_value.get('next_hops')) + + def remove(self, ni_name : str, protocol : str, prefix : str) -> None: + self._items.pop((ni_name, protocol, prefix), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, StaticRoutes.STRUCT) + +class Tables: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/table[{:s},{:s}]', ['protocol', 'address_family']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, address_family : str) -> None: + item = self._items.setdefault((ni_name, protocol, address_family), dict()) + item['protocol' ] = protocol + item['address_family'] = address_family + + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: + self._items.pop((ni_name, protocol, address_family), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Tables.STRUCT) + +class Vlans: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/vlan[{:d}]', ['vlan_id', 'name', 'members']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, ni_name : str, vlan_id : int, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, vlan_id), dict()) + item['vlan_id'] = vlan_id + item['name' ] = resource_value.get('name') + item['members'] = sorted(resource_value.get('members')) + + def remove(self, ni_name : str, vlan_id : int) -> None: + self._items.pop((ni_name, vlan_id), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Vlans.STRUCT) + +class StorageNetworkInstance: + def __init__(self) -> None: + self.network_instances = NetworkInstances() + self.interfaces = Interfaces() + self.protocols = Protocols() + self.protocol_static = StaticRoutes() + self.tables = Tables() + self.vlans = Vlans() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_NET_INST.match(resource_key) + if match is not None: + self.network_instances.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTOCOL.match(resource_key) + if match is not None: + self.protocols.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTO_STATIC.match(resource_key) + if match is not None: + self.protocol_static.add(match.group(1), match.group(2), match.group(3), resource_value) + continue + + match = RE_RESKEY_TABLE.match(resource_key) + if match is not None: + self.tables.add(match.group(1), match.group(2), match.group(3)) + continue + + match = RE_RESKEY_VLAN.match(resource_key) + if match is not None: + self.vlans.add(match.group(1), int(match.group(2)), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.network_instances.compose_resources()) + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.protocols.compose_resources()) + expected_config.extend(self.protocol_static.compose_resources()) + expected_config.extend(self.tables.compose_resources()) + expected_config.extend(self.vlans.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py new file mode 100644 index 000000000..4da48af46 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -0,0 +1,32 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Tuple + +def compose_resources( + storage : Dict[Tuple, Dict], config_struct : List[Tuple[str, List[str]]] +) -> List[Dict]: + expected_config = list() + + for resource_key_fields, resource_value_data in storage.items(): + for resource_key_template, resource_key_field_names in config_struct: + resource_key = resource_key_template.format(*resource_key_fields) + resource_value = { + field_name : resource_value_data[field_name] + for field_name in resource_key_field_names + if field_name in resource_value_data and resource_value_data[field_name] is not None + } + expected_config.append((resource_key, resource_value)) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/__init__.py b/src/device/tests/gnmi_openconfig/storage/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 47c8e1cdb..dd0561a2b 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -12,22 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import deepdiff, logging, os, pytest, re, time -from typing import Dict +import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position +import itertools, logging, pytest, time +from typing import Dict from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES ) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route -from .storage import ( # pylint: disable=unused-import - storage, # be careful, order of symbols is important here!; storage should be the first one - get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, - populate_network_instances_storage +from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.check_updates import check_updates +from .tools.expected_config_composers import ( + compose_expected_config__interface, compose_expected_config__network_instance ) +from .tools.request_composers import ( + interface, network_instance, network_instance_interface, network_instance_static_route +) +from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -56,19 +59,40 @@ def driver() -> GnmiOpenConfigDriver: _driver.Disconnect() +##### STORAGE FIXTURE ################################################################################################## + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield Storage() + + ##### NETWORK INSTANCE DETAILS ######################################################################################### -NI_NAME = 'test-l3-svc' -NI_TYPE = 'L3VRF' -NI_INTERFACES = [ - # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled - ('Ethernet1', 0, '192.168.1.1', 24, True), - ('Ethernet10', 0, '192.168.10.1', 24, True), -] -NI_STATIC_ROUTES = [ - # prefix, gateway, metric - ('172.0.0.0/24', '172.16.0.2', 1), - ('172.2.0.0/24', '172.16.0.3', 1), +NETWORK_INSTANCES = [ + { + 'name': 'test-l3-svc', + 'type': 'L3VRF', + 'interfaces': [ + {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + }, + { + 'name': 'test-l2-svc', + 'type': 'L2VSI', + 'interfaces': [ + {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + } ] @@ -76,523 +100,249 @@ NI_STATIC_ROUTES = [ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_endpoints(driver, storage) + storage.endpoints.populate(results_getconfig) + check_config_endpoints(driver, storage) def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_interfaces(driver, storage) + storage.interfaces.populate(results_getconfig) + check_config_interfaces(driver, storage) def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_network_instances(driver, storage) + storage.network_instances.populate(results_getconfig) + check_config_network_instances(driver, storage) def test_set_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance(NI_NAME, NI_TYPE), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_set.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - network_instances = sorted([NI_NAME]) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - #for resource_key, resource_value in expected_getconfig: - # if resource_key == '/network_instance[default]/vlan[1]': - # resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) - for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - interfaces = sorted([ - if_name - for if_name, _, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - interfaces = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) - for prefix, gateway, metric in NI_STATIC_ROUTES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_set = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - prefixes = sorted([ - prefix - for prefix, _, _ in NI_STATIC_ROUTES - ]) - results = set(results_setconfig) - assert len(results) == len(prefixes) - for prefix in prefixes: - assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { - 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric - }) - for prefix, gateway, metric in NI_STATIC_ROUTES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), - network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_deleteconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #expected_getconfig = get_expected_interface_config(storage) - - #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - #num_diffs = len(diff_data) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ + [ + (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ]))) - interface_ids = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_deleteconfig) - assert len(results) == len(interface_ids) - for interface_id in interface_ids: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ + [ni_if['name'] for ni_if in ni.get('interfaces', list())] + for ni in NETWORK_INSTANCES + ]))) - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure resources_to_delete = [ - network_instance(NI_NAME, 'L3VRF'), + network_instance(ni['name'], ni['type']) + for ni in NETWORK_INSTANCES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) - network_instances = sorted([NI_NAME]) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/__init__.py b/src/device/tests/gnmi_openconfig/tools/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py new file mode 100644 index 000000000..017a7038e --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -0,0 +1,82 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, deepdiff, logging, time +from typing import Callable, Dict, List, Tuple +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver +from device.tests.gnmi_openconfig.storage.Storage import Storage +from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_network_instance + +LOGGER = logging.getLogger(__name__) + +def check_expected_config( + driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + num_retry = 0 + return_data = None + while num_retry < max_retries: + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + return_data = copy.deepcopy(results_getconfig) + + results_getconfig = [ + func_adapt_returned_config(resource_key, resource_value) + for resource_key, resource_value in results_getconfig + ] + + diff_data = deepdiff.DeepDiff(sorted(expected_config), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(retry_delay) + num_retry -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + return return_data + +def check_config_endpoints( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_ENDPOINTS], storage.endpoints.get_expected_config(), + adapt_endpoint, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_interfaces( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_INTERFACES], storage.interfaces.get_expected_config(), + adapt_interface, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_network_instances( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + expected_config = + return check_expected_config( + driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), + adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay + ) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py new file mode 100644 index 000000000..7f31844cf --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -0,0 +1,21 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple + +def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: + results = set(results) + assert len(results) == len(item_ids) + for item_id in item_ids: + assert (format_str.format(*item_id), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py new file mode 100644 index 000000000..487476c01 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py @@ -0,0 +1,58 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + + +def compose_expected_config__network_instance( + network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False +) -> List[Dict]: + expected_config = list() + for network_instance in network_instances: + ni_name = network_instance['name'] + ni_type = network_instance['type'] + + expected_config.extend([ + ('/network_instance[{:s}]'.format(ni_name), { + 'name': ni_name, 'type': ni_type + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + + if include_interfaces: + expected_config.extend([ + ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { + + }) + for interface in network_instance.get('interfaces', list()) + ]) + + if include_static_routes: + expected_config.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + 'next_hop_index': 0, 'metric': static_route['metric'] + }) + for static_route in network_instance.get('static_routes', list()) + ]) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py similarity index 100% rename from src/device/tests/gnmi_openconfig/request_composers.py rename to src/device/tests/gnmi_openconfig/tools/request_composers.py diff --git a/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py new file mode 100644 index 000000000..3712f9365 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py @@ -0,0 +1,29 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, Tuple + +def adapt_endpoint(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_interface(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_network_instance(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + return resource_key, resource_value -- GitLab From 3a2f62eb760354066323ce060c835ed43e1363a2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 15:16:54 +0000 Subject: [PATCH 09/34] Device component - GNMI OpenConfig: - Updated unitary tests --- .../storage/StorageNetworkInstance.py | 11 +- .../test_unitary_gnmi_openconfig.py | 139 +++++++++--------- .../gnmi_openconfig/tools/check_config.py | 1 - .../tools/expected_config_composers.py | 58 -------- 4 files changed, 78 insertions(+), 131 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index 558cc032c..fa3364883 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -45,16 +45,14 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ('/network_instance[{:s}]/interface[{:s}]', []), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() def add(self, ni_name : str, if_name : str) -> None: - item = self._items.setdefault((ni_name, if_name), dict()) - item['ni_name'] = ni_name - item['if_name'] = if_name + self._items.setdefault((ni_name, if_name), dict()) def remove(self, ni_name : str, if_name : str) -> None: self._items.pop((ni_name, if_name), None) @@ -86,6 +84,11 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] + #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + # 'next_hop_index': 0, 'metric': static_route['metric'] + #}) + def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index dd0561a2b..3970e65a6 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -16,21 +16,17 @@ import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position -import itertools, logging, pytest, time +import logging, pytest, time from typing import Dict -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver -from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances -from .tools.check_updates import check_updates -from .tools.expected_config_composers import ( - compose_expected_config__interface, compose_expected_config__network_instance +from .storage.Storage import Storage +from .tools.check_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances ) +from .tools.check_updates import check_updates from .tools.request_composers import ( interface, network_instance, network_instance_interface, network_instance_static_route ) -from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -200,7 +196,7 @@ def test_add_interfaces_to_network_instance( subif_index = ni_if['sif_index'] resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) @@ -218,22 +214,26 @@ def test_set_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_set = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_set.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { + 'prefix': ni_sr_prefix, + }) - resources_to_set = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -246,22 +246,24 @@ def test_del_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_delete.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -274,26 +276,22 @@ def test_del_interfaces_from_network_instance( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ - [ - (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ]))) - + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -305,22 +303,24 @@ def test_del_interfaces( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - resources_to_delete = list(itertools.chain(*[ - [ - interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ - [ni_if['name'] for ni_if in ni.get('interfaces', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -333,16 +333,19 @@ def test_del_network_instances( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_delete.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.remove(ni_name) - resources_to_delete = [ - network_instance(ni['name'], ni['type']) - for ni in NETWORK_INSTANCES - ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) + check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 017a7038e..974acdeba 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -75,7 +75,6 @@ def check_config_network_instances( driver : GnmiOpenConfigDriver, storage : Storage, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: - expected_config = return check_expected_config( driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py deleted file mode 100644 index 487476c01..000000000 --- a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, List - - -def compose_expected_config__network_instance( - network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False -) -> List[Dict]: - expected_config = list() - for network_instance in network_instances: - ni_name = network_instance['name'] - ni_type = network_instance['type'] - - expected_config.extend([ - ('/network_instance[{:s}]'.format(ni_name), { - 'name': ni_name, 'type': ni_type - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - - if include_interfaces: - expected_config.extend([ - ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { - - }) - for interface in network_instance.get('interfaces', list()) - ]) - - if include_static_routes: - expected_config.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - 'next_hop_index': 0, 'metric': static_route['metric'] - }) - for static_route in network_instance.get('static_routes', list()) - ]) - - return expected_config -- GitLab From ffe63af338f70a638ec5df86c47be131ce117261 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 18:14:24 +0000 Subject: [PATCH 10/34] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/tools/check_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 974acdeba..5258da80c 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -25,10 +25,12 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) def check_expected_config( - driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: + LOGGER.info('expected_config = {:s}'.format(str(expected_config))) + num_retry = 0 return_data = None while num_retry < max_retries: -- GitLab From fce71b000b701d2d4e511092389eccb448d28987 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 12 Jan 2024 19:03:48 +0000 Subject: [PATCH 11/34] Device component - GNMI OpenConfig: - Corrected Component PORT retrieval - Corrected Static Route management - Corrected unitary tests - Added libyang to Dockerfile and corrected copy of clients - Corrected requirements.in --- src/device/Dockerfile | 21 +- src/device/requirements.in | 14 +- .../gnmi_openconfig/handlers/Component.py | 4 +- .../handlers/NetworkInstance.py | 17 +- .../handlers/NetworkInstanceStaticRoute.py | 6 +- .../storage/StorageEndpoints.py | 5 +- .../storage/StorageInterface.py | 11 +- .../storage/StorageNetworkInstance.py | 45 ++- .../tests/gnmi_openconfig/storage/Tools.py | 1 + .../test_unitary_gnmi_openconfig.py | 259 +++++++++--------- .../gnmi_openconfig/tools/check_updates.py | 5 +- .../{check_config.py => manage_config.py} | 30 +- .../tools/request_composers.py | 4 +- 13 files changed, 250 insertions(+), 172 deletions(-) rename src/device/tests/gnmi_openconfig/tools/{check_config.py => manage_config.py} (71%) diff --git a/src/device/Dockerfile b/src/device/Dockerfile index 656662552..2bcb5322a 100644 --- a/src/device/Dockerfile +++ b/src/device/Dockerfile @@ -53,6 +53,21 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install build-essential cmake libpcre2-dev python3-dev python3-cffi && \ + rm -rf /var/lib/apt/lists/* +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/device WORKDIR /var/teraflow/device @@ -62,9 +77,11 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ COPY src/device/. device/ -COPY src/monitoring/. monitoring/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ # Start the service ENTRYPOINT ["python", "-m", "device.service"] diff --git a/src/device/requirements.in b/src/device/requirements.in index d8a33455e..20ed1e2dc 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -12,30 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. - anytree==2.8.0 APScheduler==3.10.1 bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 +ipaddress Jinja2==3.0.3 +libyang==2.8.0 +macaddress ncclient==0.6.13 p4runtime==1.3.0 pandas==1.5.* paramiko==2.9.2 +pyang==2.6.* +git+https://github.com/robshakir/pyangbind.git python-json-logger==2.0.2 #pytz==2021.3 #redis==4.1.2 requests==2.27.1 requests-mock==1.9.3 -xmltodict==0.12.0 tabulate -ipaddress -macaddress -yattag -pyang==2.6.* -git+https://github.com/robshakir/pyangbind.git websockets==10.4 +xmltodict==0.12.0 +yattag # pip's dependency resolver does not take into account installed packages. # p4runtime does not specify the version of grpcio/protobuf it needs, so it tries to install latest one diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 73728192f..9b92c6b83 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging # libyang +import json, logging, re # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType from ._Handler import _Handler @@ -55,7 +55,7 @@ class ComponentHandler(_Handler): # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. - interface_name = component_name.lower().replace('-port', '') + interface_name = re.sub(r'\-[pP][oO][rR][tT]', '', component_name) endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index b97612987..d8231b2a6 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -134,18 +134,15 @@ class NetworkInstanceHandler(_Handler): for static_route in static_routes: static_route_prefix = static_route['prefix'] - next_hops = static_route.get('next-hops', {}).get('next-hop', []) - _next_hops = [ - { - 'index' : next_hop['index'], - 'gateway': next_hop['config']['next-hop'], - 'metric' : next_hop['config']['metric'], + next_hops = { + next_hop['index'] : { + 'next_hop': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], } - for next_hop in next_hops - ] - _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + for next_hop in static_route.get('next-hops', {}).get('next-hop', []) + } - _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + _static_route = {'prefix': static_route_prefix, 'next_hops': next_hops} entry_static_route_key = '{:s}/static_routes[{:s}]'.format( entry_protocol_key, static_route_prefix ) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 0343e3cba..03c04e316 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -39,8 +39,8 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' - next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_str(resource_value, 'next_hop_index') # AUTO_1_172-0-0-1 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) @@ -74,7 +74,7 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_ni_pr_sr.create_path('config/prefix', prefix) yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') - yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh_path = 'next-hop[index="{:s}"]'.format(next_hop_index) yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py index 815a1b0ad..d2596b732 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -39,13 +39,16 @@ class Endpoints: for _, field_names in Endpoints.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) item['sample_types'] = { sample_type_id : sample_type_path.format(ep_uuid) for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() } + def get(self, ep_uuid : str) -> Dict: + return self._items.get(ep_uuid) + def remove(self, ep_uuid : str) -> None: self._items.pop(ep_uuid, None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py index a0391e92f..0933433cb 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -37,7 +37,10 @@ class Interfaces: item['name'] = if_name for _, field_names in Interfaces.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) + + def get(self, if_name : str) -> Dict: + return self._items.get(if_name) def remove(self, if_name : str) -> None: self._items.pop(if_name, None) @@ -57,6 +60,9 @@ class SubInterfaces: item = self._items.setdefault((if_name, subif_index), dict()) item['index'] = subif_index + def get(self, if_name : str, subif_index : int) -> Dict: + return self._items.get((if_name, subif_index)) + def remove(self, if_name : str, subif_index : int) -> None: self._items.pop((if_name, subif_index), None) @@ -77,6 +83,9 @@ class IPv4Addresses: item['origin'] = resource_value.get('origin') item['prefix'] = resource_value.get('prefix') + def get(self, if_name : str, subif_index : int, ipv4_address : str) -> Dict: + return self._items.get((if_name, subif_index, ipv4_address)) + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: self._items.pop((if_name, subif_index, ipv4_address), None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index fa3364883..ba437ef9d 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -37,6 +37,9 @@ class NetworkInstances: item['name'] = ni_name item['type'] = resource_value.get('type') + def get(self, ni_name : str) -> Dict: + return self._items.get(ni_name) + def remove(self, ni_name : str) -> None: self._items.pop(ni_name, None) @@ -45,17 +48,24 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', []), + ('/network_instance[{:s}]/interface[{:s}.{:d}]', ['name', 'id', 'if_name', 'sif_index']), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() - def add(self, ni_name : str, if_name : str) -> None: - self._items.setdefault((ni_name, if_name), dict()) + def add(self, ni_name : str, if_name : str, sif_index : int) -> None: + item = self._items.setdefault((ni_name, if_name, sif_index), dict()) + item['name' ] = ni_name + item['id' ] = '{:s}.{:d}'.format(if_name, sif_index) + item['if_name' ] = if_name + item['sif_index'] = sif_index + + def get(self, ni_name : str, if_name : str, sif_index : int) -> Dict: + return self._items.get((ni_name, if_name, sif_index)) - def remove(self, ni_name : str, if_name : str) -> None: - self._items.pop((ni_name, if_name), None) + def remove(self, ni_name : str, if_name : str, sif_index : int) -> None: + self._items.pop((ni_name, if_name, sif_index), None) def compose_resources(self) -> List[Dict]: return compose_resources(self._items, Interfaces.STRUCT) @@ -73,6 +83,9 @@ class Protocols: item['id' ] = protocol item['name'] = protocol + def get(self, ni_name : str, protocol : str) -> Dict: + return self._items.get((ni_name, protocol)) + def remove(self, ni_name : str, protocol : str) -> None: self._items.pop((ni_name, protocol), None) @@ -84,18 +97,16 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] - #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - # 'next_hop_index': 0, 'metric': static_route['metric'] - #}) - def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: item = self._items.setdefault((ni_name, protocol, prefix), dict()) item['prefix' ] = prefix - item['next_hops'] = sorted(resource_value.get('next_hops')) + item['next_hops'] = resource_value.get('next_hops') + + def get(self, ni_name : str, protocol : str, prefix : str) -> Dict: + return self._items.get((ni_name, protocol, prefix)) def remove(self, ni_name : str, protocol : str, prefix : str) -> None: self._items.pop((ni_name, protocol, prefix), None) @@ -116,6 +127,9 @@ class Tables: item['protocol' ] = protocol item['address_family'] = address_family + def get(self, ni_name : str, protocol : str, address_family : str) -> Dict: + return self._items.get((ni_name, protocol, address_family)) + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: self._items.pop((ni_name, protocol, address_family), None) @@ -136,6 +150,9 @@ class Vlans: item['name' ] = resource_value.get('name') item['members'] = sorted(resource_value.get('members')) + def get(self, ni_name : str, vlan_id : int) -> Dict: + return self._items.get((ni_name, vlan_id)) + def remove(self, ni_name : str, vlan_id : int) -> None: self._items.pop((ni_name, vlan_id), None) @@ -160,7 +177,11 @@ class StorageNetworkInstance: match = RE_RESKEY_INTERFACE.match(resource_key) if match is not None: - self.interfaces.add(match.group(1), match.group(2)) + if_id = match.group(2) + if_id_parts = if_id.split('.') + if_name = if_id_parts[0] + sif_index = 0 if len(if_id_parts) == 1 else int(if_id_parts[1]) + self.interfaces.add(match.group(1), if_name, sif_index) continue match = RE_RESKEY_PROTOCOL.match(resource_key) diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py index 4da48af46..c9dab12e6 100644 --- a/src/device/tests/gnmi_openconfig/storage/Tools.py +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -21,6 +21,7 @@ def compose_resources( for resource_key_fields, resource_value_data in storage.items(): for resource_key_template, resource_key_field_names in config_struct: + if isinstance(resource_key_fields, (str, int, float, bool)): resource_key_fields = (resource_key_fields,) resource_key = resource_key_template.format(*resource_key_fields) resource_value = { field_name : resource_value_data[field_name] diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 3970e65a6..a601e1f23 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -17,11 +17,15 @@ os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position import logging, pytest, time -from typing import Dict +from typing import Dict, List +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver from .storage.Storage import Storage -from .tools.check_config import ( - check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.manage_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances, del_config, get_config, set_config ) from .tools.check_updates import check_updates from .tools.request_composers import ( @@ -69,26 +73,26 @@ NETWORK_INSTANCES = [ 'name': 'test-l3-svc', 'type': 'L3VRF', 'interfaces': [ - {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet1', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, ], 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, ] }, - { - 'name': 'test-l2-svc', - 'type': 'L2VSI', - 'interfaces': [ - {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, - ], - 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, - ] - } + #{ + # 'name': 'test-l2-svc', + # 'type': 'L2VSI', + # 'interfaces': [ + # {'name': 'Ethernet2', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + # {'name': 'Ethernet4', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + # ], + # 'static_routes': [ + # {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + # {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, + # ] + #} ] @@ -98,7 +102,7 @@ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_endpoints(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_ENDPOINTS]) storage.endpoints.populate(results_getconfig) check_config_endpoints(driver, storage) @@ -107,7 +111,7 @@ def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_interfaces(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_INTERFACES]) storage.interfaces.populate(results_getconfig) check_config_interfaces(driver, storage) @@ -116,7 +120,7 @@ def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_network_instances(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_NETWORK_INSTANCES]) storage.network_instances.populate(results_getconfig) check_config_network_instances(driver, storage) @@ -136,17 +140,18 @@ def test_set_network_instances( resources_to_set.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) - - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + storage.network_instances.protocols.add(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') + + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_set_interfaces( +def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -154,33 +159,24 @@ def test_set_interfaces( check_config_network_instances(driver, storage) resources_to_set = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_set.append(interface( - if_name, subif_index, ipv4_address, ipv4_prefix, enabled - )) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { - 'origin' : 'STATIC', 'prefix': ipv4_prefix - }) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/interface[{:s}]', if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_add_interfaces_to_network_instance( +def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -188,23 +184,30 @@ def test_add_interfaces_to_network_instance( check_config_network_instances(driver, storage) resources_to_set = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name in default_vlan_members: default_vlan_members.remove(if_name) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_set_network_instance_static_routes( @@ -219,24 +222,28 @@ def test_set_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_set.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_set.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocols.add(ni_name, 'STATIC') storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { - 'prefix': ni_sr_prefix, + 'prefix': ni_sr_prefix, 'next_hops': { + ni_sr_next_hop_index: {'next_hop': ni_sr_next_hop, 'metric': ni_sr_metric} + } }) + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instance_static_routes( @@ -251,79 +258,80 @@ def test_del_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_delete.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_delete.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + + storage.network_instances.protocols.remove(ni_name, 'STATIC') storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces_from_network_instance( +def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name not in default_vlan_members: default_vlan_members.append(if_name) - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces( + +def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name, subif_index) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instances( @@ -331,7 +339,7 @@ def test_del_network_instances( storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() ni_names = list() @@ -341,11 +349,12 @@ def test_del_network_instances( resources_to_delete.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.remove(ni_name) + storage.network_instances.protocols.remove(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py index 7f31844cf..a9e2a1be9 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_updates.py +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -17,5 +17,6 @@ from typing import Iterable, List, Tuple def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: results = set(results) assert len(results) == len(item_ids) - for item_id in item_ids: - assert (format_str.format(*item_id), True) in results + for item_id_fields in item_ids: + if isinstance(item_id_fields, (str, int, float, bool)): item_id_fields = (item_id_fields,) + assert (format_str.format(*item_id_fields), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/manage_config.py similarity index 71% rename from src/device/tests/gnmi_openconfig/tools/check_config.py rename to src/device/tests/gnmi_openconfig/tools/manage_config.py index 5258da80c..72d6a09d3 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/manage_config.py @@ -13,7 +13,7 @@ # limitations under the License. import copy, deepdiff, logging, time -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, List, Tuple, Union from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES @@ -24,6 +24,28 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) +def get_config(driver : GnmiOpenConfigDriver, resources_to_get : List[str]) -> List[Tuple[str, Dict]]: + LOGGER.info('[get_config] resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('[get_config] results_getconfig = {:s}'.format(str(results_getconfig))) + return results_getconfig + +def set_config( + driver : GnmiOpenConfigDriver, resources_to_set : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('[set_config] resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('[set_config] results_setconfig = {:s}'.format(str(results_setconfig))) + return results_setconfig + +def del_config( + driver : GnmiOpenConfigDriver, resources_to_delete : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + return results_deleteconfig + def check_expected_config( driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, @@ -34,9 +56,7 @@ def check_expected_config( num_retry = 0 return_data = None while num_retry < max_retries: - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + results_getconfig = get_config(driver, resources_to_get) return_data = copy.deepcopy(results_getconfig) results_getconfig = [ @@ -49,7 +69,7 @@ def check_expected_config( if num_diffs == 0: break # let the device take some time to reconfigure time.sleep(retry_delay) - num_retry -= 1 + num_retry += 1 if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 diff --git a/src/device/tests/gnmi_openconfig/tools/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py index faa8425c8..be0587101 100644 --- a/src/device/tests/gnmi_openconfig/tools/request_composers.py +++ b/src/device/tests/gnmi_openconfig/tools/request_composers.py @@ -29,10 +29,10 @@ def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: } return str_path, str_data -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: +def network_instance_static_route(ni_name, prefix, next_hop_index, next_hop, metric=1) -> Tuple[str, Dict]: str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) str_data = { - 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + 'name': ni_name, 'prefix': prefix, 'next_hop_index': next_hop_index, 'next_hop': next_hop, 'metric': metric } return str_path, str_data -- GitLab From c6f443a5d825bfda0bdaa2a34db283d30a018da9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 15 Jan 2024 13:48:14 +0000 Subject: [PATCH 12/34] Device component - GNMI OpenConfig: - Added TODO.txt --- .../service/drivers/gnmi_openconfig/TODO.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/device/service/drivers/gnmi_openconfig/TODO.txt diff --git a/src/device/service/drivers/gnmi_openconfig/TODO.txt b/src/device/service/drivers/gnmi_openconfig/TODO.txt new file mode 100644 index 000000000..ba8ff1c2c --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/TODO.txt @@ -0,0 +1,15 @@ +- update parse() @ InterfaceCounter.py +- update compose() @ NetworkInstance.py +- update compose() @ NetworkInstanceInterface.py +- implement parse() @ NetworkInstanceInterface.py +- update compose() @ NetworkInstanceStaticRoute.py +- implement parse() @ NetworkInstanceStaticRoute.py +- Fix MonitoringThread.py + + +there is an error removing static routes that makes unitary tests to crash +uncomment commented check_config_network_instance and validate + +- implement L2 VPN with BGP +- implement L3 VPN with BGP +- test static routes with ping -- GitLab From 0b89701b198e73126e8419c571ea314eb21b26c5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 22 Feb 2024 12:16:07 +0000 Subject: [PATCH 13/34] Dataplane-in-a-box: - Updated Arista cEOS image version - Updated README.md --- dataplane-in-a-box/README.md | 1 + dataplane-in-a-box/arista.clab.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 45e5dc5e0..603b24114 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -46,6 +46,7 @@ docker exec -it clab-arista-wan1 bash ## Access cEOS CLI ```bash docker exec -it clab-arista-wan1 Cli +docker exec -it clab-arista-wan2 Cli ``` ## Configure ContainerLab clients diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 4f3b77129..3a92c7e9b 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -25,7 +25,7 @@ topology: arista_ceos: kind: arista_ceos #image: ceos:4.30.4M - image: ceos:4.31.1F + image: ceos:4.31.2F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest -- GitLab From 5696254239b649a49c6155a618f7b54a928c786f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 1 Mar 2024 18:27:51 +0000 Subject: [PATCH 14/34] Dataplane-in-a-box: - Updated ContainerLab scenario - Updated TFS descriptors - Updated scripts - Updated README.md --- dataplane-in-a-box/README.md | 6 + dataplane-in-a-box/arista.clab.yml | 26 +++- dataplane-in-a-box/clab-cli-dc1.sh | 3 + dataplane-in-a-box/clab-cli-dc2.sh | 3 + .../{ceos-cli-wan1.sh => clab-cli-wan1.sh} | 0 .../{ceos-cli-wan2.sh => clab-cli-wan2.sh} | 0 dataplane-in-a-box/clab-cli-wan3.sh | 3 + dataplane-in-a-box/clab-load-image.sh | 19 --- dataplane-in-a-box/clab-pull-images.sh | 18 --- dataplane-in-a-box/links.json | 136 ------------------ .../{topology.json => tfs-01-topo-nodes.json} | 59 ++------ dataplane-in-a-box/tfs-02-topo-links.json | 63 ++++++++ dataplane-in-a-box/tfs-03-dc2dc-l2svc.json | 17 +++ ...3-service.json => tfs-04-dc2dc-l3svc.json} | 18 +-- 14 files changed, 139 insertions(+), 232 deletions(-) create mode 100755 dataplane-in-a-box/clab-cli-dc1.sh create mode 100755 dataplane-in-a-box/clab-cli-dc2.sh rename dataplane-in-a-box/{ceos-cli-wan1.sh => clab-cli-wan1.sh} (100%) rename dataplane-in-a-box/{ceos-cli-wan2.sh => clab-cli-wan2.sh} (100%) create mode 100755 dataplane-in-a-box/clab-cli-wan3.sh delete mode 100755 dataplane-in-a-box/clab-load-image.sh delete mode 100755 dataplane-in-a-box/clab-pull-images.sh delete mode 100644 dataplane-in-a-box/links.json rename dataplane-in-a-box/{topology.json => tfs-01-topo-nodes.json} (50%) create mode 100644 dataplane-in-a-box/tfs-02-topo-links.json create mode 100644 dataplane-in-a-box/tfs-03-dc2dc-l2svc.json rename dataplane-in-a-box/{dc-2-dc-l3-service.json => tfs-04-dc2dc-l3svc.json} (72%) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 603b24114..4dd22dec3 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -19,6 +19,12 @@ source dataplane-in-a-box/deploy_specs.sh sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 ``` +## Download Arista cEOS image and create Docker image +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +docker import arista/cEOS64-lab-4.31.2F.tar ceos:4.31.2F +``` + ## Deploy scenario ```bash cd ~/tfs-ctrl/dataplane-in-a-box diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 3a92c7e9b..2865100bd 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -35,21 +35,37 @@ topology: kind: arista_ceos mgmt-ipv4: 172.20.20.101 ports: [6001:6030] + wan2: kind: arista_ceos mgmt-ipv4: 172.20.20.102 ports: [6002:6030] - client1: + wan3: + kind: arista_ceos + mgmt-ipv4: 172.20.20.103 + ports: [6003:6030] + + dc1: kind: linux mgmt-ipv4: 172.20.20.201 ports: [2201:22] - client2: + exec: + - ip link set address 00:c1:ab:00:00:01 dev eth1 + - ip address add 192.168.1.10/24 dev eth1 + - ip route add 192.168.2.0/24 via 192.168.1.1 + + dc2: kind: linux mgmt-ipv4: 172.20.20.202 ports: [2202:22] + exec: + - ip link set address 00:c1:ab:00:00:02 dev eth1 + - ip address add 192.168.2.10/24 dev eth1 + - ip route add 192.168.1.0/24 via 192.168.2.1 links: - - endpoints: ["wan1:eth1", "wan2:eth1"] - - endpoints: ["client1:eth1", "wan1:eth10"] - - endpoints: ["client2:eth1", "wan2:eth10"] + - endpoints: ["wan1:eth2", "wan2:eth1"] + - endpoints: ["wan2:eth3", "wan3:eth2"] + - endpoints: ["wan1:eth10", "dc1:eth1"] + - endpoints: ["wan3:eth10", "dc2:eth1"] diff --git a/dataplane-in-a-box/clab-cli-dc1.sh b/dataplane-in-a-box/clab-cli-dc1.sh new file mode 100755 index 000000000..7d793f035 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc1 bash diff --git a/dataplane-in-a-box/clab-cli-dc2.sh b/dataplane-in-a-box/clab-cli-dc2.sh new file mode 100755 index 000000000..311d6dae5 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc2 bash diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/clab-cli-wan1.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan1.sh rename to dataplane-in-a-box/clab-cli-wan1.sh diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/clab-cli-wan2.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan2.sh rename to dataplane-in-a-box/clab-cli-wan2.sh diff --git a/dataplane-in-a-box/clab-cli-wan3.sh b/dataplane-in-a-box/clab-cli-wan3.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-wan3.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh deleted file mode 100755 index 87e666422..000000000 --- a/dataplane-in-a-box/clab-load-image.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download image from Arista account > software downloads - -cd /home/tfs/tfs-ctrl/dataplane-in-a-box -docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh deleted file mode 100755 index 8f2805c6b..000000000 --- a/dataplane-in-a-box/clab-pull-images.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker pull ghcr.io/hellt/network-multitool:latest -#docker pull ghcr.io/nokia/srlinux:23.7.2 -#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json deleted file mode 100644 index 832a24fdd..000000000 --- a/dataplane-in-a-box/links.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "contexts": [ - {"context_id": {"context_uuid": {"uuid": "admin"}}} - ], - "topologies": [ - {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} - ], - "devices": [ - { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - } - ], - "links": [ - { - "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - } - ] -} diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/tfs-01-topo-nodes.json similarity index 50% rename from dataplane-in-a-box/topology.json rename to dataplane-in-a-box/tfs-01-topo-nodes.json index 42752235d..3b5e42b99 100644 --- a/dataplane-in-a-box/topology.json +++ b/dataplane-in-a-box/tfs-01-topo-nodes.json @@ -7,85 +7,54 @@ ], "devices": [ { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc1"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc2"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "wan1"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": false - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan2"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan3"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.103"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} } - ], - "links": [] + ] } diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json new file mode 100644 index 000000000..b9070dd9d --- /dev/null +++ b/dataplane-in-a-box/tfs-02-topo-links.json @@ -0,0 +1,63 @@ +{ + "links": [ + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/1==wan1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/3==wan3/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/2==wan2/ethernet-1/3"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc1/eth1==wan1/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/10==dc1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan3/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/10==dc2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json new file mode 100644 index 000000000..8d10e5f4b --- /dev/null +++ b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l2svc"} + }, + "service_type": 2, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": []} + } + ] +} diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json similarity index 72% rename from dataplane-in-a-box/dc-2-dc-l3-service.json rename to dataplane-in-a-box/tfs-04-dc2dc-l3svc.json index cb9ef972e..b21cba0da 100644 --- a/dataplane-in-a-box/dc-2-dc-l3-service.json +++ b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json @@ -2,33 +2,33 @@ "services": [ { "service_id": { - "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l3svc"} }, "service_type": 1, "service_status": {"service_status": 1}, "service_endpoint_ids": [ - {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, - {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} ], "service_constraints": [], "service_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}} ]} -- GitLab From 083c9d0f575ad738634959a1a86e97616bed5091 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 23 Apr 2024 08:24:39 +0000 Subject: [PATCH 15/34] Device component: - added libyang version pinning to Dockerfile --- src/device/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/device/Dockerfile b/src/device/Dockerfile index ace38d934..1f9e629bb 100644 --- a/src/device/Dockerfile +++ b/src/device/Dockerfile @@ -61,6 +61,9 @@ RUN apt-get --yes --quiet --quiet update && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/libyang RUN git clone https://github.com/CESNET/libyang.git /var/libyang +WORKDIR /var/libyang +RUN git fetch +RUN git checkout v2.1.148 RUN mkdir -p /var/libyang/build WORKDIR /var/libyang/build RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. -- GitLab From 01019ca8410dbda1d4a70b17d6f42795f0211d6b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 23 Apr 2024 08:25:42 +0000 Subject: [PATCH 16/34] Device component - gNMI OpenConfig Driver: - Added yang module "openconfig-bgp-types" - Enabled some debug log messages --- .../drivers/gnmi_openconfig/GnmiSessionHandler.py | 10 ++++++---- .../drivers/gnmi_openconfig/handlers/YangHandler.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index dd542c0dc..9c2c0abb3 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -95,22 +95,24 @@ class GnmiSessionHandler: str_resource_name = 'resource_key[#{:d}]'.format(i) try: chk_string(str_resource_name, resource_key, allow_empty=False) - #self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) + self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) str_path = get_path(resource_key) - #self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) + self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) get_request.path.append(path_from_string(str_path)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception parsing {:s}: {:s}' self._logger.exception(MSG.format(str_resource_name, str(resource_key))) parsing_results.append((resource_key, e)) # if validation fails, store the exception + self._logger.debug('parsing_results={:s}'.format(str(parsing_results))) + if len(parsing_results) > 0: return parsing_results metadata = [('username', self._username), ('password', self._password)] timeout = None # GNMI_SUBSCRIPTION_TIMEOUT = int(sampling_duration) get_reply = self._stub.Get(get_request, metadata=metadata, timeout=timeout) - #self._logger.info('get_reply={:s}'.format(grpc_message_to_json_string(get_reply))) + self._logger.debug('get_reply={:s}'.format(grpc_message_to_json_string(get_reply))) results = [] #results[str_filter] = [i, None, False] # (index, value, processed?) @@ -127,7 +129,7 @@ class GnmiSessionHandler: # resource_key_tuple[2] = True for update in notification.update: - #self._logger.info('update={:s}'.format(grpc_message_to_json_string(update))) + self._logger.debug('update={:s}'.format(grpc_message_to_json_string(update))) str_path = path_to_string(update.path) #resource_key_tuple = results.get(str_path) #if resource_key_tuple is None: diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py index fe8672187..32a15ca2e 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py @@ -23,6 +23,7 @@ YANG_SEARCH_PATHS = ':'.join([ YANG_MODULES = [ 'iana-if-type', + 'openconfig-bgp-types', 'openconfig-vlan-types', 'openconfig-interfaces', -- GitLab From 4ce70dafea4fd0754e2ebe0e7b0b8ae91f394f25 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 23 Apr 2024 08:29:00 +0000 Subject: [PATCH 17/34] DataPlane-in-a-box: - Simplified scenario to 2 devices - Disabled monitoring and service monitors in TFS deploy specs --- dataplane-in-a-box/arista.clab.yml | 12 ++---- dataplane-in-a-box/clab-cli-wan3.sh | 3 -- dataplane-in-a-box/deploy_specs.sh | 4 +- dataplane-in-a-box/example_config/wan1.conf | 37 +++++++++++++++++++ dataplane-in-a-box/example_config/wan2.conf | 37 +++++++++++++++++++ dataplane-in-a-box/tfs-01-topo-nodes.json | 10 ----- dataplane-in-a-box/tfs-02-topo-links.json | 41 +++++---------------- 7 files changed, 89 insertions(+), 55 deletions(-) delete mode 100755 dataplane-in-a-box/clab-cli-wan3.sh create mode 100644 dataplane-in-a-box/example_config/wan1.conf create mode 100644 dataplane-in-a-box/example_config/wan2.conf diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 2865100bd..fa4957f19 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -41,11 +41,6 @@ topology: mgmt-ipv4: 172.20.20.102 ports: [6002:6030] - wan3: - kind: arista_ceos - mgmt-ipv4: 172.20.20.103 - ports: [6003:6030] - dc1: kind: linux mgmt-ipv4: 172.20.20.201 @@ -65,7 +60,8 @@ topology: - ip route add 192.168.1.0/24 via 192.168.2.1 links: - - endpoints: ["wan1:eth2", "wan2:eth1"] - - endpoints: ["wan2:eth3", "wan3:eth2"] + - endpoints: ["wan1:eth1", "wan2:eth1"] + - endpoints: ["wan1:eth2", "wan2:eth2"] + - endpoints: ["wan1:eth3", "wan2:eth3"] - endpoints: ["wan1:eth10", "dc1:eth1"] - - endpoints: ["wan3:eth10", "dc2:eth1"] + - endpoints: ["wan2:eth10", "dc2:eth1"] diff --git a/dataplane-in-a-box/clab-cli-wan3.sh b/dataplane-in-a-box/clab-cli-wan3.sh deleted file mode 100755 index c931ac940..000000000 --- a/dataplane-in-a-box/clab-cli-wan3.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker exec -it clab-arista-wan2 Cli diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh index 1a978e3a9..93d5b2b82 100755 --- a/dataplane-in-a-box/deploy_specs.sh +++ b/dataplane-in-a-box/deploy_specs.sh @@ -24,7 +24,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring -export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate ZTP #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" @@ -54,7 +54,7 @@ export TFS_K8S_NAMESPACE="tfs" export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" # Uncomment to monitor performance of components -export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" # Uncomment when deploying Optical CyberSecurity #export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" diff --git a/dataplane-in-a-box/example_config/wan1.conf b/dataplane-in-a-box/example_config/wan1.conf new file mode 100644 index 000000000..fccb7b43f --- /dev/null +++ b/dataplane-in-a-box/example_config/wan1.conf @@ -0,0 +1,37 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable +configure +ip routing + +interface Ethernet1 + no switchport + ip address 10.1.2.1/30 +exit + +interface Loopback0 + ip address 10.0.0.1/32 +exit + +interface Ethernet10 + no switchport + ip address 192.168.1.1/24 +exit + +router bgp 65001 + router-id 10.0.0.1 + neighbor 10.1.2.2 remote-as 65001 + network 192.168.1.0/24 +exit diff --git a/dataplane-in-a-box/example_config/wan2.conf b/dataplane-in-a-box/example_config/wan2.conf new file mode 100644 index 000000000..6edc0c405 --- /dev/null +++ b/dataplane-in-a-box/example_config/wan2.conf @@ -0,0 +1,37 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable +configure +ip routing + +interface Ethernet1 + no switchport + ip address 10.1.2.2/30 +exit + +interface Loopback0 + ip address 10.0.0.2/32 +exit + +interface Ethernet10 + no switchport + ip address 192.168.2.1/24 +exit + +router bgp 65001 + router-id 10.0.0.2 + neighbor 10.1.2.1 remote-as 65001 + network 192.168.2.0/24 +exit diff --git a/dataplane-in-a-box/tfs-01-topo-nodes.json b/dataplane-in-a-box/tfs-01-topo-nodes.json index 3b5e42b99..d0fafbfe8 100644 --- a/dataplane-in-a-box/tfs-01-topo-nodes.json +++ b/dataplane-in-a-box/tfs-01-topo-nodes.json @@ -45,16 +45,6 @@ "username": "admin", "password": "admin", "use_tls": false }}} ]} - }, - { - "device_id": {"device_uuid": {"uuid": "wan3"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.103"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": false - }}} - ]} } ] } diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json index b9070dd9d..78765a7d5 100644 --- a/dataplane-in-a-box/tfs-02-topo-links.json +++ b/dataplane-in-a-box/tfs-02-topo-links.json @@ -1,31 +1,23 @@ { "links": [ { - "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/1"}}, + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/1==wan2/ethernet-1/1"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} ] }, { - "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/1==wan1/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/3==wan3/ethernet-1/2"}}, + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/2"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, - {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} ] }, { - "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/2==wan2/ethernet-1/3"}}, + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/3==wan2/ethernet-1/3"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}} ] }, @@ -38,25 +30,10 @@ ] }, { - "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/10==dc1/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, - {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan3/ethernet-1/10"}}, + "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan2/ethernet-1/10"}}, "link_endpoint_ids": [ {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/10==dc2/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, - {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} ] } ] -- GitLab From 1546aafacbcbba4a8387fd75a2d264f59dea71c4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 13:53:59 +0000 Subject: [PATCH 18/34] Common - Tools - Object Factory: - Added field "name" to json_endpoint() builder method --- src/common/tools/object_factory/EndPoint.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/tools/object_factory/EndPoint.py b/src/common/tools/object_factory/EndPoint.py index 85a5d4494..faf1accd3 100644 --- a/src/common/tools/object_factory/EndPoint.py +++ b/src/common/tools/object_factory/EndPoint.py @@ -43,13 +43,14 @@ def json_endpoint_ids( def json_endpoint( device_id : Dict, endpoint_uuid : str, endpoint_type : str, topology_id : Optional[Dict] = None, - kpi_sample_types : List[int] = [], location : Optional[Dict] = None + name : Optional[str] = None, kpi_sample_types : List[int] = [], location : Optional[Dict] = None ): result = { 'endpoint_id': json_endpoint_id(device_id, endpoint_uuid, topology_id=topology_id), 'endpoint_type': endpoint_type, } + if name is not None: result['name'] = name if kpi_sample_types is not None and len(kpi_sample_types) > 0: result['kpi_sample_types'] = copy.deepcopy(kpi_sample_types) if location is not None: -- GitLab From 0ba504e3f0b33e304e2c2b74a40186459fc8c4fa Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 13:55:08 +0000 Subject: [PATCH 19/34] Service component - Service Handler API: - Added method get_service_settings() into SettingsHandler --- src/service/service/service_handler_api/SettingsHandler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/service/service/service_handler_api/SettingsHandler.py b/src/service/service/service_handler_api/SettingsHandler.py index 293de54aa..24c5b638a 100644 --- a/src/service/service/service_handler_api/SettingsHandler.py +++ b/src/service/service/service_handler_api/SettingsHandler.py @@ -57,6 +57,11 @@ class SettingsHandler: def get(self, key_or_path : Union[str, List[str]], default : Optional[Any] = None) -> Optional[TreeNode]: return get_subnode(self.__resolver, self.__config, key_or_path, default=default) + def get_service_settings(self) -> Optional[TreeNode]: + service_settings_uri = '/settings' + service_settings = self.get(service_settings_uri) + return service_settings + def get_device_settings(self, device : Device) -> Optional[TreeNode]: device_keys = device.device_id.device_uuid.uuid, device.name -- GitLab From a31cab6ba31b7658a711acf437939433abc45dab Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 14:02:34 +0000 Subject: [PATCH 20/34] Service component - L3NM gNMI OpenConfig Service Handler: - Added StaticRouteGenerator class - ConfigRuleComposer: added dump methods to composer classes - ConfigRuleComposer: extended configure methods to load previous configurations in devices - ConfigRuleComposer: generalized to a single network instance for all static routing-based services - ConfigRuleComposer: Updated static route organization with indexes and metrics - ConfigRuleComposer: Added tracking of connected networks - ConfigRuleComposer: Corrected config rule composer methods - Integrated StaticRouteGenerator into Service Handler - Multiple cosmetic improvements - Added code for a future unitary test --- .../ConfigRuleComposer.py | 176 ++++++++++++++--- .../L3NMGnmiOpenConfigServiceHandler.py | 44 +++-- .../StaticRouteGenerator.py | 183 ++++++++++++++++++ .../MockServiceHandler.py | 160 +++++++++++++++ .../MockTaskExecutor.py | 57 ++++++ .../__init__.py | 14 ++ .../test_unitary.py | 147 ++++++++++++++ 7 files changed, 734 insertions(+), 47 deletions(-) create mode 100644 src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py create mode 100644 src/service/tests/test_l3nm_gnmi_static_rule_gen/MockServiceHandler.py create mode 100644 src/service/tests/test_l3nm_gnmi_static_rule_gen/MockTaskExecutor.py create mode 100644 src/service/tests/test_l3nm_gnmi_static_rule_gen/__init__.py create mode 100644 src/service/tests/test_l3nm_gnmi_static_rule_gen/test_unitary.py diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py index be314a8c1..5db2c5b2f 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py @@ -12,33 +12,62 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Tuple -from common.proto.context_pb2 import Device, EndPoint +import json, netaddr, re +from typing import Dict, List, Optional, Set, Tuple +from common.DeviceTypes import DeviceTypeEnum +from common.proto.context_pb2 import ConfigActionEnum, Device, EndPoint, Service from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set - from service.service.service_handler_api.AnyTreeTools import TreeNode -def _interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = {'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, - 'sub_if_enabled': enabled, 'sub_if_ipv4_enabled': enabled, - 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix} - return str_path, str_data +NETWORK_INSTANCE = 'teraflowsdn' + +RE_IF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') +RE_SR = re.compile(r'^\/network_instance\[([^\]]+)\]\/protocols\[STATIC\]/route\[ ([^\:]+)\:([^\]]+)\]$') + +def _interface( + interface : str, if_type : Optional[str] = 'l3ipvlan', index : int = 0, vlan_id : Optional[int] = None, + address_ip : Optional[str] = None, address_prefix : Optional[int] = None, mtu : Optional[int] = None, + enabled : bool = True +) -> Tuple[str, Dict]: + path = '/interface[{:s}]/subinterface[{:d}]'.format(interface, index) + data = {'name': interface, 'type': if_type, 'index': index, 'enabled': enabled} + if if_type is not None: data['type'] = if_type + if vlan_id is not None: data['vlan_id'] = vlan_id + if address_ip is not None: data['address_ip'] = address_ip + if address_prefix is not None: data['address_prefix'] = address_prefix + if mtu is not None: data['mtu'] = mtu + return path, data -def _network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data +def _network_instance(ni_name : str, ni_type : str) -> Tuple[str, Dict]: + path = '/network_instance[{:s}]'.format(ni_name) + data = {'name': ni_name, 'type': ni_type} + return path, data -def _network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data +def _network_instance_protocol(ni_name : str, protocol : str) -> Tuple[str, Dict]: + path = '/network_instance[{:s}]/protocols[{:s}]'.format(ni_name, protocol) + data = {'name': ni_name, 'identifier': protocol, 'protocol_name': protocol} + return path, data -def _network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data +def _network_instance_protocol_static(ni_name : str) -> Tuple[str, Dict]: + return _network_instance_protocol(ni_name, 'STATIC') + +def _network_instance_protocol_static_route( + ni_name : str, prefix : str, next_hop : str, index : int = 0, metric : Optional[int] = None +) -> Tuple[str, Dict]: + protocol = 'STATIC' + path = '/network_instance[{:s}]/protocols[{:s}]/static_route[{:s}:{:d}]'.format(ni_name, protocol, prefix, index) + data = { + 'name': ni_name, 'identifier': protocol, 'protocol_name': protocol, + 'prefix': prefix, 'index': index, 'next_hop': next_hop + } + if metric is not None: data['metric'] = metric + return path, data + +def _network_instance_interface(ni_name : str, interface : str, sub_interface_index : int) -> Tuple[str, Dict]: + sub_interface_name = '{:s}.{:d}'.format(interface, sub_interface_index) + path = '/network_instance[{:s}]/interface[{:s}]'.format(ni_name, sub_interface_name) + data = {'name': ni_name, 'id': sub_interface_name, 'interface': interface, 'subinterface': sub_interface_index} + return path, data class EndpointComposer: def __init__(self, endpoint_uuid : str) -> None: @@ -46,33 +75,47 @@ class EndpointComposer: self.objekt : Optional[EndPoint] = None self.sub_interface_index = 0 self.ipv4_address = None - self.ipv4_prefix = None + self.ipv4_prefix_len = None - def configure(self, endpoint_obj : EndPoint, settings : Optional[TreeNode]) -> None: - self.objekt = endpoint_obj + def configure(self, endpoint_obj : Optional[EndPoint], settings : Optional[TreeNode]) -> None: + if endpoint_obj is not None: + self.objekt = endpoint_obj if settings is None: return json_settings : Dict = settings.value self.ipv4_address = json_settings['ipv4_address'] - self.ipv4_prefix = json_settings['ipv4_prefix'] + self.ipv4_prefix_len = json_settings['ipv4_prefix_len'] self.sub_interface_index = json_settings['sub_interface_index'] def get_config_rules(self, network_instance_name : str, delete : bool = False) -> List[Dict]: + if self.ipv4_address is None: return [] + if self.ipv4_prefix_len is None: return [] json_config_rule = json_config_rule_delete if delete else json_config_rule_set return [ json_config_rule(*_interface( - self.objekt.name, self.sub_interface_index, self.ipv4_address, self.ipv4_prefix, True + self.objekt.name, index=self.sub_interface_index, address_ip=self.ipv4_address, + address_prefix=self.ipv4_prefix_len, enabled=True )), json_config_rule(*_network_instance_interface( network_instance_name, self.objekt.name, self.sub_interface_index )), ] + def dump(self) -> Dict: + return { + 'sub_interface_index' : self.sub_interface_index, + 'ipv4_address' : self.ipv4_address, + 'ipv4_prefix_len' : self.ipv4_prefix_len, + } + class DeviceComposer: def __init__(self, device_uuid : str) -> None: self.uuid = device_uuid self.objekt : Optional[Device] = None self.endpoints : Dict[str, EndpointComposer] = dict() - self.static_routes : Dict[str, str] = dict() + self.connected : Set[str] = set() + + # {prefix => {index => (next_hop, metric)}} + self.static_routes : Dict[str, Dict[int, Tuple[str, Optional[int]]]] = dict() def get_endpoint(self, endpoint_uuid : str) -> EndpointComposer: if endpoint_uuid not in self.endpoints: @@ -81,39 +124,108 @@ class DeviceComposer: def configure(self, device_obj : Device, settings : Optional[TreeNode]) -> None: self.objekt = device_obj + for endpoint_obj in device_obj.device_endpoints: + self.get_endpoint(endpoint_obj.name).configure(endpoint_obj, None) + + for config_rule in device_obj.device_config.config_rules: + if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue + if config_rule.WhichOneof('config_rule') != 'custom': continue + config_rule_custom = config_rule.custom + + match = RE_IF.match(config_rule_custom.resource_key) + if match is not None: + if_name, subif_index = match.groups() + resource_value = json.loads(config_rule_custom.resource_value) + ipv4_network = str(resource_value['address_ip']) + ipv4_prefix_len = int(resource_value['address_prefix']) + endpoint = self.get_endpoint(if_name) + endpoint.ipv4_address = ipv4_network + endpoint.ipv4_prefix_len = ipv4_prefix_len + endpoint.sub_interface_index = int(subif_index) + endpoint_ip_network = netaddr.IPNetwork('{:s}/{:d}'.format(ipv4_network, ipv4_prefix_len)) + self.connected.add(str(endpoint_ip_network.cidr)) + + match = RE_SR.match(config_rule_custom.resource_key) + if match is not None: + ni_name, prefix, index = match.groups() + if ni_name != NETWORK_INSTANCE: continue + resource_value : Dict = json.loads(config_rule_custom.resource_value) + next_hop = resource_value['next_hop'] + metric = resource_value.get('metric') + self.static_routes.setdefault(prefix, dict())[index] = (next_hop, metric) + if settings is None: return json_settings : Dict = settings.value - static_routes = json_settings.get('static_routes', []) + static_routes : List[Dict] = json_settings.get('static_routes', []) for static_route in static_routes: prefix = static_route['prefix'] + index = static_route.get('index', 0) next_hop = static_route['next_hop'] - self.static_routes[prefix] = next_hop + metric = static_route.get('metric') + self.static_routes.setdefault(prefix, dict())[index] = (next_hop, metric) def get_config_rules(self, network_instance_name : str, delete : bool = False) -> List[Dict]: + SELECTED_DEVICES = {DeviceTypeEnum.PACKET_ROUTER.value, DeviceTypeEnum.EMULATED_PACKET_ROUTER.value} + if self.objekt.device_type not in SELECTED_DEVICES: return [] + json_config_rule = json_config_rule_delete if delete else json_config_rule_set config_rules = [ json_config_rule(*_network_instance(network_instance_name, 'L3VRF')) ] for endpoint in self.endpoints.values(): config_rules.extend(endpoint.get_config_rules(network_instance_name, delete=delete)) - for prefix, next_hop in self.static_routes.items(): + if len(self.static_routes) > 0: config_rules.append( - json_config_rule(*_network_instance_static_route(network_instance_name, prefix, next_hop)) + json_config_rule(*_network_instance_protocol_static(network_instance_name)) ) + for prefix, indexed_static_rule in self.static_routes.items(): + for index, (next_hop, metric) in indexed_static_rule.items(): + config_rules.append( + json_config_rule(*_network_instance_protocol_static_route( + network_instance_name, prefix, next_hop, index=index, metric=metric + )) + ) if delete: config_rules = list(reversed(config_rules)) return config_rules + def dump(self) -> Dict: + return { + 'endpoints' : { + endpoint_uuid : endpoint.dump() + for endpoint_uuid, endpoint in self.endpoints.items() + }, + 'connected' : list(self.connected), + 'static_routes' : self.static_routes, + } + class ConfigRuleComposer: def __init__(self) -> None: + self.objekt : Optional[Service] = None self.devices : Dict[str, DeviceComposer] = dict() + def configure(self, service_obj : Service, settings : Optional[TreeNode]) -> None: + self.objekt = service_obj + if settings is None: return + #json_settings : Dict = settings.value + # For future use + def get_device(self, device_uuid : str) -> DeviceComposer: if device_uuid not in self.devices: self.devices[device_uuid] = DeviceComposer(device_uuid) return self.devices[device_uuid] - def get_config_rules(self, network_instance_name : str, delete : bool = False) -> Dict[str, List[Dict]]: + def get_config_rules( + self, network_instance_name : str = NETWORK_INSTANCE, delete : bool = False + ) -> Dict[str, List[Dict]]: return { device_uuid : device.get_config_rules(network_instance_name, delete=delete) for device_uuid, device in self.devices.items() } + + def dump(self) -> Dict: + return { + 'devices' : { + device_uuid : device.dump() + for device_uuid, device in self.devices.items() + } + } diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py index 5856b5f61..9142c9d1e 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py @@ -18,11 +18,12 @@ from common.method_wrappers.Decorator import MetricsPool, metered_subclass_metho from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type -from service.service.service_handler_api.Tools import get_device_endpoint_uuids, get_endpoint_matching from service.service.service_handler_api._ServiceHandler import _ServiceHandler from service.service.service_handler_api.SettingsHandler import SettingsHandler +from service.service.service_handler_api.Tools import get_device_endpoint_uuids, get_endpoint_matching from service.service.task_scheduler.TaskExecutor import TaskExecutor from .ConfigRuleComposer import ConfigRuleComposer +from .StaticRouteGenerator import StaticRouteGenerator LOGGER = logging.getLogger(__name__) @@ -35,16 +36,22 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): self.__service = service self.__task_executor = task_executor self.__settings_handler = SettingsHandler(service.service_config, **settings) - self.__composer = ConfigRuleComposer() - self.__endpoint_map : Dict[Tuple[str, str], str] = dict() + self.__config_rule_composer = ConfigRuleComposer() + self.__static_route_generator = StaticRouteGenerator(self.__config_rule_composer) + self.__endpoint_map : Dict[Tuple[str, str], Tuple[str, str]] = dict() def _compose_config_rules(self, endpoints : List[Tuple[str, str, Optional[str]]]) -> None: + if len(endpoints) % 2 != 0: raise Exception('Number of endpoints should be even') + + service_settings = self.__settings_handler.get_service_settings() + self.__config_rule_composer.configure(self.__service, service_settings) + for endpoint in endpoints: device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) device_settings = self.__settings_handler.get_device_settings(device_obj) - _device = self.__composer.get_device(device_obj.name) + _device = self.__config_rule_composer.get_device(device_obj.name) _device.configure(device_obj, device_settings) endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) @@ -52,7 +59,9 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): _endpoint = _device.get_endpoint(endpoint_obj.name) _endpoint.configure(endpoint_obj, endpoint_settings) - self.__endpoint_map[(device_uuid, endpoint_uuid)] = device_obj.name + self.__endpoint_map[(device_uuid, endpoint_uuid)] = (device_obj.name, endpoint_obj.name) + + self.__static_route_generator.compose(endpoints) def _do_configurations( self, config_rules_per_device : Dict[str, List[Dict]], endpoints : List[Tuple[str, str, Optional[str]]], @@ -62,7 +71,7 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): results_per_device = dict() for device_name,json_config_rules in config_rules_per_device.items(): try: - device_obj = self.__composer.get_device(device_name).objekt + device_obj = self.__config_rule_composer.get_device(device_name).objekt if len(json_config_rules) == 0: continue del device_obj.device_config.config_rules[:] for json_config_rule in json_config_rules: @@ -78,7 +87,8 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): results = [] for endpoint in endpoints: device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) - device_name = self.__endpoint_map[(device_uuid, endpoint_uuid)] + device_name, _ = self.__endpoint_map[(device_uuid, endpoint_uuid)] + if device_name not in results_per_device: continue results.append(results_per_device[device_name]) return results @@ -88,12 +98,14 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): ) -> List[Union[bool, Exception]]: chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] - service_uuid = self.__service.service_id.service_uuid.uuid - #settings = self.__settings_handler.get('/settings') + #service_uuid = self.__service.service_id.service_uuid.uuid self._compose_config_rules(endpoints) - network_instance_name = service_uuid.split('-')[0] - config_rules_per_device = self.__composer.get_config_rules(network_instance_name, delete=False) + #network_instance_name = service_uuid.split('-')[0] + #config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=False) + config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=False) + LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device))) results = self._do_configurations(config_rules_per_device, endpoints) + LOGGER.debug('results={:s}'.format(str(results))) return results @metered_subclass_method(METRICS_POOL) @@ -102,12 +114,14 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): ) -> List[Union[bool, Exception]]: chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] - service_uuid = self.__service.service_id.service_uuid.uuid - #settings = self.__settings_handler.get('/settings') + #service_uuid = self.__service.service_id.service_uuid.uuid self._compose_config_rules(endpoints) - network_instance_name = service_uuid.split('-')[0] - config_rules_per_device = self.__composer.get_config_rules(network_instance_name, delete=True) + #network_instance_name = service_uuid.split('-')[0] + #config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=True) + config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=True) + LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device))) results = self._do_configurations(config_rules_per_device, endpoints, delete=True) + LOGGER.debug('results={:s}'.format(str(results))) return results @metered_subclass_method(METRICS_POOL) diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py new file mode 100644 index 000000000..6479a07fe --- /dev/null +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py @@ -0,0 +1,183 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, logging, netaddr +from typing import List, Optional, Tuple +from .ConfigRuleComposer import ConfigRuleComposer + +LOGGER = logging.getLogger(__name__) + +# Used to infer routing networks for adjacent ports when there is no hint in device/endpoint settings +ROOT_NEIGHBOR_ROUTING_NETWORK = netaddr.IPNetwork('10.254.254.0/16') +NEIGHBOR_ROUTING_NETWORKS_PREFIX_LEN = 30 +NEIGHBOR_ROUTING_NETWORKS = set(ROOT_NEIGHBOR_ROUTING_NETWORK.subnet(NEIGHBOR_ROUTING_NETWORKS_PREFIX_LEN)) + +def _generate_neighbor_addresses() -> Tuple[netaddr.IPAddress, netaddr.IPAddress, int]: + ip_network = NEIGHBOR_ROUTING_NETWORKS.pop() + ip_addresses = list(ip_network.iter_hosts()) + ip_addresses.append(NEIGHBOR_ROUTING_NETWORKS_PREFIX_LEN) + return ip_addresses + +def _compute_gateway(ip_network : netaddr.IPNetwork, gateway_host=1) -> netaddr.IPAddress: + return netaddr.IPAddress(ip_network.cidr.first + gateway_host) + +def _compose_ipv4_network(ipv4_network, ipv4_prefix_len) -> netaddr.IPNetwork: + return netaddr.IPNetwork('{:s}/{:d}'.format(str(ipv4_network), int(ipv4_prefix_len))) + +class StaticRouteGenerator: + def __init__(self, config_rule_composer : ConfigRuleComposer) -> None: + self._config_rule_composer = config_rule_composer + + def compose(self, connection_hop_list : List[Tuple[str, str, Optional[str]]]) -> None: + link_endpoints = self._compute_link_endpoints(connection_hop_list) + LOGGER.debug('link_endpoints = {:s}'.format(str(link_endpoints))) + + self._compute_link_addresses(link_endpoints) + LOGGER.debug('config_rule_composer = {:s}'.format(json.dumps(self._config_rule_composer.dump()))) + + self._discover_connected_networks(connection_hop_list) + LOGGER.debug('config_rule_composer = {:s}'.format(json.dumps(self._config_rule_composer.dump()))) + + # Compute and propagate static routes forward (service_endpoint_a => service_endpoint_b) + self._compute_static_routes(link_endpoints) + + # Compute and propagate static routes backward (service_endpoint_b => service_endpoint_a) + reversed_endpoints = list(reversed(connection_hop_list)) + reversed_link_endpoints = self._compute_link_endpoints(reversed_endpoints) + LOGGER.debug('reversed_link_endpoints = {:s}'.format(str(reversed_link_endpoints))) + self._compute_static_routes(reversed_link_endpoints) + + LOGGER.debug('config_rule_composer = {:s}'.format(json.dumps(self._config_rule_composer.dump()))) + + def _compute_link_endpoints( + self, connection_hop_list : List[Tuple[str, str, Optional[str]]] + ) -> List[Tuple[Tuple[str, str, Optional[str]], Tuple[str, str, Optional[str]]]]: + num_connection_hops = len(connection_hop_list) + if num_connection_hops % 2 != 0: raise Exception('Number of connection hops must be even') + if num_connection_hops < 4: raise Exception('Number of connection hops must be >= 4') + + # Skip service endpoints (first and last) + it_connection_hops = iter(connection_hop_list[1:-1]) + return list(zip(it_connection_hops, it_connection_hops)) + + def _compute_link_addresses( + self, link_endpoints_list : List[Tuple[Tuple[str, str, Optional[str]], Tuple[str, str, Optional[str]]]] + ) -> None: + for link_endpoints in link_endpoints_list: + device_endpoint_a, device_endpoint_b = link_endpoints + + device_uuid_a, endpoint_uuid_a = device_endpoint_a[0:2] + endpoint_a = self._config_rule_composer.get_device(device_uuid_a).get_endpoint(endpoint_uuid_a) + + device_uuid_b, endpoint_uuid_b = device_endpoint_b[0:2] + endpoint_b = self._config_rule_composer.get_device(device_uuid_b).get_endpoint(endpoint_uuid_b) + + if endpoint_a.ipv4_address is None and endpoint_b.ipv4_address is None: + ip_endpoint_a, ip_endpoint_b, prefix_len = _generate_neighbor_addresses() + endpoint_a.ipv4_address = str(ip_endpoint_a) + endpoint_a.ipv4_prefix_len = prefix_len + endpoint_b.ipv4_address = str(ip_endpoint_b) + endpoint_b.ipv4_prefix_len = prefix_len + elif endpoint_a.ipv4_address is not None and endpoint_b.ipv4_address is None: + prefix_len = endpoint_a.ipv4_prefix_len + ip_network_a = _compose_ipv4_network(endpoint_a.ipv4_address, prefix_len) + if prefix_len > 30: + MSG = 'Unsupported prefix_len for {:s}: {:s}' + raise Exception(MSG.format(str(endpoint_a), str(prefix_len))) + ip_endpoint_b = _compute_gateway(ip_network_a, gateway_host=1) + if ip_endpoint_b == ip_network_a.ip: + ip_endpoint_b = _compute_gateway(ip_network_a, gateway_host=2) + endpoint_b.ipv4_address = str(ip_endpoint_b) + endpoint_b.ipv4_prefix_len = prefix_len + elif endpoint_a.ipv4_address is None and endpoint_b.ipv4_address is not None: + prefix_len = endpoint_b.ipv4_prefix_len + ip_network_b = _compose_ipv4_network(endpoint_b.ipv4_address, prefix_len) + if prefix_len > 30: + MSG = 'Unsupported prefix_len for {:s}: {:s}' + raise Exception(MSG.format(str(endpoint_b), str(prefix_len))) + ip_endpoint_a = _compute_gateway(ip_network_b, gateway_host=1) + if ip_endpoint_a == ip_network_b.ip: + ip_endpoint_a = _compute_gateway(ip_network_b, gateway_host=2) + endpoint_a.ipv4_address = str(ip_endpoint_a) + endpoint_a.ipv4_prefix_len = prefix_len + elif endpoint_a.ipv4_address is not None and endpoint_b.ipv4_address is not None: + ip_network_a = _compose_ipv4_network(endpoint_a.ipv4_address, endpoint_a.ipv4_prefix_len) + ip_network_b = _compose_ipv4_network(endpoint_b.ipv4_address, endpoint_b.ipv4_prefix_len) + if ip_network_a.cidr != ip_network_b.cidr: + MSG = 'Incompatible CIDRs: endpoint_a({:s})=>{:s} endpoint_b({:s})=>{:s}' + raise Exception(MSG.format(str(endpoint_a), str(ip_network_a), str(endpoint_b), str(ip_network_b))) + if ip_network_a.ip == ip_network_b.ip: + MSG = 'Duplicated IP: endpoint_a({:s})=>{:s} endpoint_b({:s})=>{:s}' + raise Exception(MSG.format(str(endpoint_a), str(ip_network_a), str(endpoint_b), str(ip_network_b))) + + def _discover_connected_networks(self, connection_hop_list : List[Tuple[str, str, Optional[str]]]) -> None: + for connection_hop in connection_hop_list: + device_uuid, endpoint_uuid = connection_hop[0:2] + device = self._config_rule_composer.get_device(device_uuid) + endpoint = device.get_endpoint(endpoint_uuid) + + if endpoint.ipv4_address is None: continue + ip_network = _compose_ipv4_network(endpoint.ipv4_address, endpoint.ipv4_prefix_len) + + device.connected.add(str(ip_network.cidr)) + + def _compute_static_routes( + self, link_endpoints_list : List[Tuple[Tuple[str, str, Optional[str]], Tuple[str, str, Optional[str]]]] + ) -> None: + for link_endpoints in link_endpoints_list: + device_endpoint_a, device_endpoint_b = link_endpoints + + device_uuid_a, endpoint_uuid_a = device_endpoint_a[0:2] + device_a = self._config_rule_composer.get_device(device_uuid_a) + endpoint_a = device_a.get_endpoint(endpoint_uuid_a) + + device_uuid_b, endpoint_uuid_b = device_endpoint_b[0:2] + device_b = self._config_rule_composer.get_device(device_uuid_b) + endpoint_b = device_b.get_endpoint(endpoint_uuid_b) + + # Compute static routes from networks connected in device_a + for ip_network_a in device_a.connected: + if ip_network_a in device_b.connected: continue + if ip_network_a in device_b.static_routes: continue + if ip_network_a in ROOT_NEIGHBOR_ROUTING_NETWORK: continue + endpoint_a_ip_network = _compose_ipv4_network(endpoint_a.ipv4_address, endpoint_a.ipv4_prefix_len) + next_hop = str(endpoint_a_ip_network.ip) + device_b.static_routes.setdefault(ip_network_a, dict())[0] = (next_hop, None) + + # Compute static routes from networks connected in device_b + for ip_network_b in device_b.connected: + if ip_network_b in device_a.connected: continue + if ip_network_b in device_a.static_routes: continue + if ip_network_b in ROOT_NEIGHBOR_ROUTING_NETWORK: continue + endpoint_b_ip_network = _compose_ipv4_network(endpoint_b.ipv4_address, endpoint_b.ipv4_prefix_len) + next_hop = str(endpoint_b_ip_network.ip) + device_a.static_routes.setdefault(ip_network_b, dict())[0] = (next_hop, None) + + # Propagate static routes from networks connected in device_a + for ip_network_a in device_a.static_routes.keys(): + if ip_network_a in device_b.connected: continue + if ip_network_a in device_b.static_routes: continue + if ip_network_a in ROOT_NEIGHBOR_ROUTING_NETWORK: continue + endpoint_a_ip_network = _compose_ipv4_network(endpoint_a.ipv4_address, endpoint_a.ipv4_prefix_len) + next_hop = str(endpoint_a_ip_network.ip) + device_b.static_routes.setdefault(ip_network_a, dict())[0] = (next_hop, None) + + # Propagate static routes from networks connected in device_b + for ip_network_b in device_b.static_routes.keys(): + if ip_network_b in device_a.connected: continue + if ip_network_b in device_a.static_routes: continue + if ip_network_b in ROOT_NEIGHBOR_ROUTING_NETWORK: continue + endpoint_b_ip_network = _compose_ipv4_network(endpoint_b.ipv4_address, endpoint_b.ipv4_prefix_len) + next_hop = str(endpoint_b_ip_network.ip) + device_a.static_routes.setdefault(ip_network_b, dict())[0] = (next_hop, None) diff --git a/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockServiceHandler.py b/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockServiceHandler.py new file mode 100644 index 000000000..9b3f76566 --- /dev/null +++ b/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockServiceHandler.py @@ -0,0 +1,160 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, logging +from typing import Any, Dict, List, Optional, Tuple, Union +from common.proto.context_pb2 import ConfigRule, DeviceId, Service +from common.tools.object_factory.Device import json_device_id +from common.type_checkers.Checkers import chk_type +from service.service.service_handler_api._ServiceHandler import _ServiceHandler +from service.service.service_handler_api.SettingsHandler import SettingsHandler +from service.service.service_handler_api.Tools import get_device_endpoint_uuids, get_endpoint_matching +from .MockTaskExecutor import MockTaskExecutor +from service.service.service_handlers.l3nm_gnmi_openconfig.ConfigRuleComposer import ConfigRuleComposer +from service.service.service_handlers.l3nm_gnmi_openconfig.StaticRouteGenerator import StaticRouteGenerator + +LOGGER = logging.getLogger(__name__) + +class MockServiceHandler(_ServiceHandler): + def __init__( # pylint: disable=super-init-not-called + self, service : Service, task_executor : MockTaskExecutor, **settings + ) -> None: + self.__service = service + self.__task_executor = task_executor + self.__settings_handler = SettingsHandler(service.service_config, **settings) + self.__config_rule_composer = ConfigRuleComposer() + self.__static_route_generator = StaticRouteGenerator(self.__config_rule_composer) + self.__endpoint_map : Dict[Tuple[str, str], Tuple[str, str]] = dict() + + def _compose_config_rules(self, endpoints : List[Tuple[str, str, Optional[str]]]) -> None: + if len(endpoints) % 2 != 0: raise Exception('Number of endpoints should be even') + + service_settings = self.__settings_handler.get_service_settings() + self.__config_rule_composer.configure(self.__service, service_settings) + + for endpoint in endpoints: + device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) + + device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_settings = self.__settings_handler.get_device_settings(device_obj) + _device = self.__config_rule_composer.get_device(device_obj.name) + _device.configure(device_obj, device_settings) + + endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) + endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) + _endpoint = _device.get_endpoint(endpoint_obj.name) + _endpoint.configure(endpoint_obj, endpoint_settings) + + self.__endpoint_map[(device_uuid, endpoint_uuid)] = (device_obj.name, endpoint_obj.name) + + self.__static_route_generator.compose(endpoints) + + def _do_configurations( + self, config_rules_per_device : Dict[str, List[Dict]], endpoints : List[Tuple[str, str, Optional[str]]], + delete : bool = False + ) -> List[Union[bool, Exception]]: + # Configuration is done atomically on each device, all OK / all KO per device + results_per_device = dict() + for device_name,json_config_rules in config_rules_per_device.items(): + try: + device_obj = self.__config_rule_composer.get_device(device_name).objekt + if len(json_config_rules) == 0: continue + del device_obj.device_config.config_rules[:] + for json_config_rule in json_config_rules: + device_obj.device_config.config_rules.append(ConfigRule(**json_config_rule)) + self.__task_executor.configure_device(device_obj) + results_per_device[device_name] = True + except Exception as e: # pylint: disable=broad-exception-caught + verb = 'deconfigure' if delete else 'configure' + MSG = 'Unable to {:s} Device({:s}) : ConfigRules({:s})' + LOGGER.exception(MSG.format(verb, str(device_name), str(json_config_rules))) + results_per_device[device_name] = e + + results = [] + for endpoint in endpoints: + device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) + device_name, _ = self.__endpoint_map[(device_uuid, endpoint_uuid)] + if device_name not in results_per_device: continue + results.append(results_per_device[device_name]) + return results + + def SetEndpoint( + self, endpoints : List[Tuple[str, str, Optional[str]]], connection_uuid : Optional[str] = None + ) -> List[Union[bool, Exception]]: + chk_type('endpoints', endpoints, list) + if len(endpoints) == 0: return [] + self._compose_config_rules(endpoints) + config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=False) + LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device))) + results = self._do_configurations(config_rules_per_device, endpoints) + LOGGER.debug('results={:s}'.format(str(results))) + return results + + def DeleteEndpoint( + self, endpoints : List[Tuple[str, str, Optional[str]]], connection_uuid : Optional[str] = None + ) -> List[Union[bool, Exception]]: + chk_type('endpoints', endpoints, list) + if len(endpoints) == 0: return [] + self._compose_config_rules(endpoints) + config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=True) + LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device))) + results = self._do_configurations(config_rules_per_device, endpoints, delete=True) + LOGGER.debug('results={:s}'.format(str(results))) + return results + + def SetConstraint(self, constraints : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + chk_type('constraints', constraints, list) + if len(constraints) == 0: return [] + + msg = '[SetConstraint] Method not implemented. Constraints({:s}) are being ignored.' + LOGGER.warning(msg.format(str(constraints))) + return [True for _ in range(len(constraints))] + + def DeleteConstraint(self, constraints : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + chk_type('constraints', constraints, list) + if len(constraints) == 0: return [] + + msg = '[DeleteConstraint] Method not implemented. Constraints({:s}) are being ignored.' + LOGGER.warning(msg.format(str(constraints))) + return [True for _ in range(len(constraints))] + + def SetConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + chk_type('resources', resources, list) + if len(resources) == 0: return [] + + results = [] + for resource in resources: + try: + resource_value = json.loads(resource[1]) + self.__settings_handler.set(resource[0], resource_value) + results.append(True) + except Exception as e: # pylint: disable=broad-except + LOGGER.exception('Unable to SetConfig({:s})'.format(str(resource))) + results.append(e) + + return results + + def DeleteConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + chk_type('resources', resources, list) + if len(resources) == 0: return [] + + results = [] + for resource in resources: + try: + self.__settings_handler.delete(resource[0]) + except Exception as e: # pylint: disable=broad-except + LOGGER.exception('Unable to DeleteConfig({:s})'.format(str(resource))) + results.append(e) + + return results diff --git a/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockTaskExecutor.py b/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockTaskExecutor.py new file mode 100644 index 000000000..765b04477 --- /dev/null +++ b/src/service/tests/test_l3nm_gnmi_static_rule_gen/MockTaskExecutor.py @@ -0,0 +1,57 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from enum import Enum +from typing import Dict, Optional, Union +from common.method_wrappers.ServiceExceptions import NotFoundException +from common.proto.context_pb2 import Connection, Device, DeviceId, Service +from service.service.tools.ObjectKeys import get_device_key + +LOGGER = logging.getLogger(__name__) + +CacheableObject = Union[Connection, Device, Service] + +class CacheableObjectType(Enum): + CONNECTION = 'connection' + DEVICE = 'device' + SERVICE = 'service' + +class MockTaskExecutor: + def __init__(self) -> None: + self._grpc_objects_cache : Dict[str, CacheableObject] = dict() + + # ----- Common methods --------------------------------------------------------------------------------------------- + + def _load_grpc_object(self, object_type : CacheableObjectType, object_key : str) -> Optional[CacheableObject]: + object_key = '{:s}:{:s}'.format(object_type.value, object_key) + return self._grpc_objects_cache.get(object_key) + + def _store_grpc_object(self, object_type : CacheableObjectType, object_key : str, grpc_object) -> None: + object_key = '{:s}:{:s}'.format(object_type.value, object_key) + self._grpc_objects_cache[object_key] = grpc_object + + def _delete_grpc_object(self, object_type : CacheableObjectType, object_key : str) -> None: + object_key = '{:s}:{:s}'.format(object_type.value, object_key) + self._grpc_objects_cache.pop(object_key, None) + + def get_device(self, device_id : DeviceId) -> Device: + device_key = get_device_key(device_id) + device = self._load_grpc_object(CacheableObjectType.DEVICE, device_key) + if device is None: raise NotFoundException('Device', device_key) + return device + + def configure_device(self, device : Device) -> None: + device_key = get_device_key(device.device_id) + self._store_grpc_object(CacheableObjectType.DEVICE, device_key, device) diff --git a/src/service/tests/test_l3nm_gnmi_static_rule_gen/__init__.py b/src/service/tests/test_l3nm_gnmi_static_rule_gen/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/service/tests/test_l3nm_gnmi_static_rule_gen/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/service/tests/test_l3nm_gnmi_static_rule_gen/test_unitary.py b/src/service/tests/test_l3nm_gnmi_static_rule_gen/test_unitary.py new file mode 100644 index 000000000..43709b036 --- /dev/null +++ b/src/service/tests/test_l3nm_gnmi_static_rule_gen/test_unitary.py @@ -0,0 +1,147 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run with: +# $ PYTHONPATH=./src python -m service.tests.test_l3nm_gnmi_static_rule_gen.test_unitary + +import logging +from typing import List, Optional, Tuple +from common.DeviceTypes import DeviceTypeEnum +from common.proto.context_pb2 import Device, DeviceOperationalStatusEnum, Service +from common.tools.object_factory.ConfigRule import json_config_rule_set +from common.tools.object_factory.Device import json_device, json_device_id +from common.tools.object_factory.EndPoint import json_endpoint, json_endpoint_id +from common.tools.object_factory.Service import json_service_l3nm_planned +from .MockServiceHandler import MockServiceHandler +from .MockTaskExecutor import CacheableObjectType, MockTaskExecutor + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) + +SERVICE_DC1_DC2 = Service(**json_service_l3nm_planned( + 'svc-dc1-dc2-uuid', + endpoint_ids=[ + json_endpoint_id(json_device_id('DC1'), 'int'), + json_endpoint_id(json_device_id('DC2'), 'int'), + ], + config_rules=[ + json_config_rule_set('/device[DC1]/endpoint[eth0]/settings', { + 'ipv4_address': '192.168.10.10', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + json_config_rule_set('/device[R1]/endpoint[1/2]/settings', { + 'ipv4_address': '10.0.1.1', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + #json_config_rule_set('/device[R2]/endpoint[1/2]/settings', { + # 'ipv4_address': '10.0.2.1', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + #}), + json_config_rule_set('/device[DC2]/endpoint[eth0]/settings', { + 'ipv4_address': '192.168.20.10', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + ] +)) + +SERVICE_DC1_DC3 = Service(**json_service_l3nm_planned( + 'svc-dc1-dc3-uuid', + endpoint_ids=[ + json_endpoint_id(json_device_id('DC1'), 'int'), + json_endpoint_id(json_device_id('DC3'), 'int'), + ], + config_rules=[ + json_config_rule_set('/device[DC1]/endpoint[eth0]/settings', { + 'ipv4_address': '192.168.10.10', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + #json_config_rule_set('/device[R1]/endpoint[1/2]/settings', { + # 'ipv4_address': '10.0.1.1', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + #}), + json_config_rule_set('/device[R4]/endpoint[1/1]/settings', { + 'ipv4_address': '10.0.4.1', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + json_config_rule_set('/device[DC3]/endpoint[eth0]/settings', { + 'ipv4_address': '192.168.30.10', 'ipv4_prefix_len': 24, 'sub_interface_index': 0 + }), + ] +)) + +CONNECTION_ENDPOINTS_DC1_DC2 : List[Tuple[str, str, Optional[str]]] = [ + ('DC1', 'int', None), ('DC1', 'eth0', None), + ('R1', '1/1', None), ('R1', '1/2', None), + ('R2', '1/1', None), ('R2', '1/2', None), + ('R3', '1/1', None), ('R3', '1/2', None), + ('DC2', 'eth0', None), ('DC2', 'int', None), +] + +CONNECTION_ENDPOINTS_DC1_DC3 : List[Tuple[str, str, Optional[str]]] = [ + ('DC1', 'int', None), ('DC1', 'eth0', None), + ('R1', '1/1', None), ('R1', '1/2', None), + ('R2', '1/1', None), ('R2', '1/3', None), + ('R4', '1/1', None), ('R4', '1/2', None), + ('DC3', 'eth0', None), ('DC3', 'int', None), +] + +def test_l3nm_gnmi_static_rule_gen() -> None: + dev_op_st_enabled = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + mock_task_executor = MockTaskExecutor() + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'DC1', Device(**json_device( + 'uuid-DC1', DeviceTypeEnum.EMULATED_DATACENTER.value, dev_op_st_enabled, name='DC1', endpoints=[ + json_endpoint(json_device_id('uuid-DC1'), 'uuid-int', 'packet', name='int' ), + json_endpoint(json_device_id('uuid-DC1'), 'uuid-eth0', 'packet', name='eth0'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'DC2', Device(**json_device( + 'uuid-DC2', DeviceTypeEnum.EMULATED_DATACENTER.value, dev_op_st_enabled, name='DC2', endpoints=[ + json_endpoint(json_device_id('uuid-DC2'), 'uuid-int', 'packet', name='int' ), + json_endpoint(json_device_id('uuid-DC2'), 'uuid-eth0', 'packet', name='eth0'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'DC3', Device(**json_device( + 'uuid-DC3', DeviceTypeEnum.EMULATED_DATACENTER.value, dev_op_st_enabled, name='DC3', endpoints=[ + json_endpoint(json_device_id('uuid-DC3'), 'uuid-int', 'packet', name='int' ), + json_endpoint(json_device_id('uuid-DC3'), 'uuid-eth0', 'packet', name='eth0'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'R1', Device(**json_device( + 'uuid-R1', DeviceTypeEnum.EMULATED_PACKET_ROUTER.value, dev_op_st_enabled, name='R1', endpoints=[ + json_endpoint(json_device_id('uuid-R1'), 'uuid-1/1', 'packet', name='1/1'), + json_endpoint(json_device_id('uuid-R1'), 'uuid-1/2', 'packet', name='1/2'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'R2', Device(**json_device( + 'uuid-R2', DeviceTypeEnum.EMULATED_PACKET_ROUTER.value, dev_op_st_enabled, name='R2', endpoints=[ + json_endpoint(json_device_id('uuid-R2'), 'uuid-1/1', 'packet', name='1/1'), + json_endpoint(json_device_id('uuid-R2'), 'uuid-1/2', 'packet', name='1/2'), + json_endpoint(json_device_id('uuid-R2'), 'uuid-1/3', 'packet', name='1/3'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'R3', Device(**json_device( + 'uuid-R3', DeviceTypeEnum.EMULATED_PACKET_ROUTER.value, dev_op_st_enabled, name='R3', endpoints=[ + json_endpoint(json_device_id('uuid-R3'), 'uuid-1/1', 'packet', name='1/1'), + json_endpoint(json_device_id('uuid-R3'), 'uuid-1/2', 'packet', name='1/2'), + ] + ))) + mock_task_executor._store_grpc_object(CacheableObjectType.DEVICE, 'R4', Device(**json_device( + 'uuid-R4', DeviceTypeEnum.EMULATED_PACKET_ROUTER.value, dev_op_st_enabled, name='R4', endpoints=[ + json_endpoint(json_device_id('uuid-R4'), 'uuid-1/1', 'packet', name='1/1'), + json_endpoint(json_device_id('uuid-R4'), 'uuid-1/2', 'packet', name='1/2'), + ] + ))) + + mock_service_handler = MockServiceHandler(SERVICE_DC1_DC2, mock_task_executor) + mock_service_handler.SetEndpoint(CONNECTION_ENDPOINTS_DC1_DC2) + + mock_service_handler = MockServiceHandler(SERVICE_DC1_DC3, mock_task_executor) + mock_service_handler.SetEndpoint(CONNECTION_ENDPOINTS_DC1_DC3) + +if __name__ == '__main__': + test_l3nm_gnmi_static_rule_gen() -- GitLab From 2b2ade7ce06c98623ee870f147eb3825cb9f3667 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 17:14:59 +0000 Subject: [PATCH 21/34] Service component - L3NM gNMI OpenConfig Service Handler: - Corrected static_routes structure and metrics management --- .../ConfigRuleComposer.py | 30 +++++++--------- .../StaticRouteGenerator.py | 34 +++++++++++++------ 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py index 5db2c5b2f..42747a1ae 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py @@ -22,7 +22,7 @@ from service.service.service_handler_api.AnyTreeTools import TreeNode NETWORK_INSTANCE = 'teraflowsdn' RE_IF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') -RE_SR = re.compile(r'^\/network_instance\[([^\]]+)\]\/protocols\[STATIC\]/route\[ ([^\:]+)\:([^\]]+)\]$') +RE_SR = re.compile(r'^\/network_instance\[([^\]]+)\]\/protocols\[STATIC\]/route\[([^\:]+)\:([^\]]+)\]$') def _interface( interface : str, if_type : Optional[str] = 'l3ipvlan', index : int = 0, vlan_id : Optional[int] = None, @@ -52,15 +52,15 @@ def _network_instance_protocol_static(ni_name : str) -> Tuple[str, Dict]: return _network_instance_protocol(ni_name, 'STATIC') def _network_instance_protocol_static_route( - ni_name : str, prefix : str, next_hop : str, index : int = 0, metric : Optional[int] = None + ni_name : str, prefix : str, next_hop : str, metric : int ) -> Tuple[str, Dict]: protocol = 'STATIC' - path = '/network_instance[{:s}]/protocols[{:s}]/static_route[{:s}:{:d}]'.format(ni_name, protocol, prefix, index) + path = '/network_instance[{:s}]/protocols[{:s}]/static_route[{:s}:{:d}]'.format(ni_name, protocol, prefix, metric) + index = 'AUTO_{:d}_{:s}'.format(metric, next_hop.replace('.', '-')) data = { 'name': ni_name, 'identifier': protocol, 'protocol_name': protocol, - 'prefix': prefix, 'index': index, 'next_hop': next_hop + 'prefix': prefix, 'index': index, 'next_hop': next_hop, 'metric': metric } - if metric is not None: data['metric'] = metric return path, data def _network_instance_interface(ni_name : str, interface : str, sub_interface_index : int) -> Tuple[str, Dict]: @@ -113,9 +113,7 @@ class DeviceComposer: self.objekt : Optional[Device] = None self.endpoints : Dict[str, EndpointComposer] = dict() self.connected : Set[str] = set() - - # {prefix => {index => (next_hop, metric)}} - self.static_routes : Dict[str, Dict[int, Tuple[str, Optional[int]]]] = dict() + self.static_routes : Dict[str, Dict[int, str]] = dict() # {prefix => {metric => next_hop}} def get_endpoint(self, endpoint_uuid : str) -> EndpointComposer: if endpoint_uuid not in self.endpoints: @@ -147,22 +145,20 @@ class DeviceComposer: match = RE_SR.match(config_rule_custom.resource_key) if match is not None: - ni_name, prefix, index = match.groups() + ni_name, prefix, metric = match.groups() if ni_name != NETWORK_INSTANCE: continue resource_value : Dict = json.loads(config_rule_custom.resource_value) next_hop = resource_value['next_hop'] - metric = resource_value.get('metric') - self.static_routes.setdefault(prefix, dict())[index] = (next_hop, metric) + self.static_routes.setdefault(prefix, dict())[metric] = next_hop if settings is None: return json_settings : Dict = settings.value static_routes : List[Dict] = json_settings.get('static_routes', []) for static_route in static_routes: prefix = static_route['prefix'] - index = static_route.get('index', 0) next_hop = static_route['next_hop'] - metric = static_route.get('metric') - self.static_routes.setdefault(prefix, dict())[index] = (next_hop, metric) + metric = static_route.get('metric', 0) + self.static_routes.setdefault(prefix, dict())[metric] = next_hop def get_config_rules(self, network_instance_name : str, delete : bool = False) -> List[Dict]: SELECTED_DEVICES = {DeviceTypeEnum.PACKET_ROUTER.value, DeviceTypeEnum.EMULATED_PACKET_ROUTER.value} @@ -178,11 +174,11 @@ class DeviceComposer: config_rules.append( json_config_rule(*_network_instance_protocol_static(network_instance_name)) ) - for prefix, indexed_static_rule in self.static_routes.items(): - for index, (next_hop, metric) in indexed_static_rule.items(): + for prefix, metric_next_hop in self.static_routes.items(): + for metric, next_hop in metric_next_hop.items(): config_rules.append( json_config_rule(*_network_instance_protocol_static_route( - network_instance_name, prefix, next_hop, index=index, metric=metric + network_instance_name, prefix, next_hop, metric )) ) if delete: config_rules = list(reversed(config_rules)) diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py index 6479a07fe..a16e4d5b1 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/StaticRouteGenerator.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging, netaddr +import json, logging, netaddr, sys from typing import List, Optional, Tuple from .ConfigRuleComposer import ConfigRuleComposer @@ -153,7 +153,8 @@ class StaticRouteGenerator: if ip_network_a in ROOT_NEIGHBOR_ROUTING_NETWORK: continue endpoint_a_ip_network = _compose_ipv4_network(endpoint_a.ipv4_address, endpoint_a.ipv4_prefix_len) next_hop = str(endpoint_a_ip_network.ip) - device_b.static_routes.setdefault(ip_network_a, dict())[0] = (next_hop, None) + metric = 1 + device_b.static_routes.setdefault(ip_network_a, dict())[metric] = next_hop # Compute static routes from networks connected in device_b for ip_network_b in device_b.connected: @@ -162,22 +163,35 @@ class StaticRouteGenerator: if ip_network_b in ROOT_NEIGHBOR_ROUTING_NETWORK: continue endpoint_b_ip_network = _compose_ipv4_network(endpoint_b.ipv4_address, endpoint_b.ipv4_prefix_len) next_hop = str(endpoint_b_ip_network.ip) - device_a.static_routes.setdefault(ip_network_b, dict())[0] = (next_hop, None) + metric = 1 + device_a.static_routes.setdefault(ip_network_b, dict())[metric] = next_hop # Propagate static routes from networks connected in device_a - for ip_network_a in device_a.static_routes.keys(): + for ip_network_a, metric_next_hop in device_a.static_routes.items(): if ip_network_a in device_b.connected: continue - if ip_network_a in device_b.static_routes: continue if ip_network_a in ROOT_NEIGHBOR_ROUTING_NETWORK: continue endpoint_a_ip_network = _compose_ipv4_network(endpoint_a.ipv4_address, endpoint_a.ipv4_prefix_len) - next_hop = str(endpoint_a_ip_network.ip) - device_b.static_routes.setdefault(ip_network_a, dict())[0] = (next_hop, None) + if ip_network_a in device_b.static_routes: + current_metric = min(device_b.static_routes[ip_network_a].keys()) + else: + current_metric = int(sys.float_info.max) + for metric, next_hop in metric_next_hop.items(): + new_metric = metric + 1 + if new_metric >= current_metric: continue + next_hop_a = str(endpoint_a_ip_network.ip) + device_b.static_routes.setdefault(ip_network_a, dict())[metric] = next_hop_a # Propagate static routes from networks connected in device_b for ip_network_b in device_b.static_routes.keys(): if ip_network_b in device_a.connected: continue - if ip_network_b in device_a.static_routes: continue if ip_network_b in ROOT_NEIGHBOR_ROUTING_NETWORK: continue endpoint_b_ip_network = _compose_ipv4_network(endpoint_b.ipv4_address, endpoint_b.ipv4_prefix_len) - next_hop = str(endpoint_b_ip_network.ip) - device_a.static_routes.setdefault(ip_network_b, dict())[0] = (next_hop, None) + if ip_network_b in device_a.static_routes: + current_metric = min(device_a.static_routes[ip_network_b].keys()) + else: + current_metric = int(sys.float_info.max) + for metric, next_hop in metric_next_hop.items(): + new_metric = metric + 1 + if new_metric >= current_metric: continue + next_hop_b = str(endpoint_b_ip_network.ip) + device_a.static_routes.setdefault(ip_network_b, dict())[metric] = next_hop_b -- GitLab From 578c898f0501de0ab070f07de1f0a5df44130490 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 17:16:24 +0000 Subject: [PATCH 22/34] Device - gNMI OpenConfig Driver: - Corrected Interface composition - Corrected Network Instance static route parsing - Corrected Network Instance static route composition Corrected Network Instance interface composition --- .../gnmi_openconfig/handlers/Interface.py | 38 ++++++++++--------- .../handlers/NetworkInstance.py | 29 +++++++------- .../handlers/NetworkInstanceInterface.py | 22 ++++++----- .../handlers/NetworkInstanceStaticRoute.py | 7 +++- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 6de048303..a769692ba 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -21,14 +21,14 @@ from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) class InterfaceHandler(_Handler): - def get_resource_key(self) -> str: return '/interface' + def get_resource_key(self) -> str: return '/interface/subinterface' def get_path(self) -> str: return '/openconfig-interfaces:interfaces' def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - if_name = get_str(resource_value, 'name' ) # ethernet-1/1 - sif_index = get_int(resource_value, 'sub_if_index', 0) # 0 + if_name = get_str(resource_value, 'name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'index', 0) # 0 if delete: PATH_TMPL = '/interfaces/interface[name={:s}]/subinterfaces/subinterface[index={:d}]' @@ -36,38 +36,40 @@ class InterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - if_enabled = get_bool(resource_value, 'enabled', True) # True/False - sif_enabled = get_bool(resource_value, 'sub_if_enabled', True) # True/False - sif_vlan_id = get_int (resource_value, 'sif_vlan_id', ) # 127 - sif_ipv4_enabled = get_bool(resource_value, 'sub_if_ipv4_enabled', True) # True/False - sif_ipv4_address = get_str (resource_value, 'sub_if_ipv4_address' ) # 172.16.0.1 - sif_ipv4_prefix = get_int (resource_value, 'sub_if_ipv4_prefix' ) # 24 + enabled = get_bool(resource_value, 'enabled', True) # True/False + #if_type = get_str (resource_value, 'type' ) # 'l3ipvlan' + vlan_id = get_int (resource_value, 'vlan_id', ) # 127 + address_ip = get_str (resource_value, 'address_ip' ) # 172.16.0.1 + address_prefix = get_int (resource_value, 'address_prefix') # 24 + mtu = get_int (resource_value, 'mtu' ) # 1500 yang_ifs : libyang.DContainer = yang_handler.get_data_path('/openconfig-interfaces:interfaces') yang_if_path = 'interface[name="{:s}"]'.format(if_name) yang_if : libyang.DContainer = yang_ifs.create_path(yang_if_path) yang_if.create_path('config/name', if_name ) - if if_enabled is not None: yang_if.create_path('config/enabled', if_enabled) + if enabled is not None: yang_if.create_path('config/enabled', enabled) + if mtu is not None: yang_if.create_path('config/mtu', mtu) yang_sifs : libyang.DContainer = yang_if.create_path('subinterfaces') yang_sif_path = 'subinterface[index="{:d}"]'.format(sif_index) yang_sif : libyang.DContainer = yang_sifs.create_path(yang_sif_path) yang_sif.create_path('config/index', sif_index) - if sif_enabled is not None: yang_sif.create_path('config/enabled', sif_enabled) + if enabled is not None: yang_sif.create_path('config/enabled', enabled) - if sif_vlan_id is not None: + if vlan_id is not None: yang_subif_vlan : libyang.DContainer = yang_sif.create_path('openconfig-vlan:vlan') - yang_subif_vlan.create_path('match/single-tagged/config/vlan-id', sif_vlan_id) + yang_subif_vlan.create_path('match/single-tagged/config/vlan-id', vlan_id) yang_ipv4 : libyang.DContainer = yang_sif.create_path('openconfig-if-ip:ipv4') - if sif_ipv4_enabled is not None: yang_ipv4.create_path('config/enabled', sif_ipv4_enabled) + if enabled is not None: yang_ipv4.create_path('config/enabled', enabled) - if sif_ipv4_address is not None: + if address_ip is not None and address_prefix is not None: yang_ipv4_addrs : libyang.DContainer = yang_ipv4.create_path('addresses') - yang_ipv4_addr_path = 'address[ip="{:s}"]'.format(sif_ipv4_address) + yang_ipv4_addr_path = 'address[ip="{:s}"]'.format(address_ip) yang_ipv4_addr : libyang.DContainer = yang_ipv4_addrs.create_path(yang_ipv4_addr_path) - yang_ipv4_addr.create_path('config/ip', sif_ipv4_address) - yang_ipv4_addr.create_path('config/prefix-length', sif_ipv4_prefix ) + yang_ipv4_addr.create_path('config/ip', address_ip) + yang_ipv4_addr.create_path('config/prefix-length', address_prefix) + if mtu is not None: yang_ipv4_addr.create_path('config/mtu', mtu) str_path = '/interfaces/interface[name={:s}]'.format(if_name) str_data = yang_if.print_mem('json') diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 88eccf325..c4a1c03e4 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, libyang, logging, operator +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler from .Tools import get_str @@ -125,28 +125,27 @@ class NetworkInstanceHandler(_Handler): ni_protocol_id = ni_protocol['identifier'].split(':')[-1] ni_protocol_name = ni_protocol['name'] - _protocol = {'id': ni_protocol_id, 'name': ni_protocol_name} - entry_protocol_key = '{:s}/protocol[{:s}]'.format(entry_net_inst_key, ni_protocol_id) + _protocol = {'name': ni_name, 'identifier': ni_protocol_id, 'protocol_name': ni_protocol_name} + entry_protocol_key = '{:s}/protocols[{:s}]'.format(entry_net_inst_key, ni_protocol_id) entries.append((entry_protocol_key, _protocol)) if ni_protocol_id == 'STATIC': static_routes = ni_protocol.get('static-routes', {}).get('static', []) for static_route in static_routes: static_route_prefix = static_route['prefix'] - - next_hops = { - next_hop['index'] : { + for next_hop in static_route.get('next-hops', {}).get('next-hop', []): + static_route_metric = next_hop['config']['metric'] + _static_route = { + 'prefix' : static_route_prefix, + 'index' : next_hop['index'], 'next_hop': next_hop['config']['next-hop'], - 'metric' : next_hop['config']['metric'], + 'metric' : static_route_metric, } - for next_hop in static_route.get('next-hops', {}).get('next-hop', []) - } - - _static_route = {'prefix': static_route_prefix, 'next_hops': next_hops} - entry_static_route_key = '{:s}/static_routes[{:s}]'.format( - entry_protocol_key, static_route_prefix - ) - entries.append((entry_static_route_key, _static_route)) + _static_route.update(_protocol) + entry_static_route_key = '{:s}/static_route[{:s}:{:d}]'.format( + entry_protocol_key, static_route_prefix, static_route_metric + ) + entries.append((entry_static_route_key, _static_route)) ni_tables = network_instance.get('tables', {}).get('table', []) for ni_table in ni_tables: diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index c1c59b926..ed5e04087 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -29,20 +29,22 @@ class NetworkInstanceInterfaceHandler(_Handler): def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - ni_name = get_str(resource_value, 'name' ) # test-svc - if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 - sif_index = get_int(resource_value, 'sif_index', 0) # 0 - if_id = '{:s}.{:d}'.format(if_name, sif_index) + ni_name = get_str(resource_value, 'name' ) # test-svc + ni_if_id = get_str(resource_value, 'id' ) # ethernet-1/1.0 + if_name = get_str(resource_value, 'interface' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'subinterface', 0) # 0 + + if IS_CEOS: ni_if_id = if_name if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' - str_path = PATH_TMPL.format(ni_name, if_id) + str_path = PATH_TMPL.format(ni_name, ni_if_id) str_data = json.dumps({}) return str_path, str_data - if IS_CEOS: if_id = if_name - - str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) + str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format( + ni_name, ni_if_id + ) #str_data = json.dumps({ # 'id': if_id, # 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, @@ -51,9 +53,9 @@ class NetworkInstanceInterfaceHandler(_Handler): yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) yang_ni_ifs : libyang.DContainer = yang_ni.create_path('interfaces') - yang_ni_if_path = 'interface[id="{:s}"]'.format(if_id) + yang_ni_if_path = 'interface[id="{:s}"]'.format(ni_if_id) yang_ni_if : libyang.DContainer = yang_ni_ifs.create_path(yang_ni_if_path) - yang_ni_if.create_path('config/id', if_id) + yang_ni_if.create_path('config/id', ni_if_id) yang_ni_if.create_path('config/interface', if_name) yang_ni_if.create_path('config/subinterface', sif_index) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 498cd5aee..cc561f37d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -21,8 +21,9 @@ from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) class NetworkInstanceStaticRouteHandler(_Handler): - def get_resource_key(self) -> str: return '/network_instance/static_route' - def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/static_route' + def get_resource_key(self) -> str: return '/network_instance/protocols/static_route' + def get_path(self) -> str: + return '/openconfig-network-instance:network-instances/network-instance/protocols/protocol/static-routes' def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False @@ -40,6 +41,7 @@ class NetworkInstanceStaticRouteHandler(_Handler): return str_path, str_data next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + metric = get_int(resource_value, 'metric' ) # 20 next_hop_index = get_str(resource_value, 'next_hop_index') # AUTO_1_172-0-0-1 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' @@ -78,6 +80,7 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) + yang_ni_pr_sr_nh.create_path('config/metric', metric) str_data = yang_ni_pr.print_mem('json') LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) -- GitLab From f3d6ba96980f058021afeb1eb7c9dfc6e41eac0f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Sat, 11 May 2024 17:18:26 +0000 Subject: [PATCH 23/34] DataPlane-in-a-box: - Corrected TFS descriptors - Corrected scripts - Corrected containerlab descriptor - Corrected README.md --- dataplane-in-a-box/README.md | 8 ++-- dataplane-in-a-box/arista.clab.yml | 24 +++++------- dataplane-in-a-box/clab-cli-r1.sh | 3 ++ dataplane-in-a-box/clab-cli-r2.sh | 3 ++ dataplane-in-a-box/clab-cli-wan1.sh | 3 -- dataplane-in-a-box/clab-cli-wan2.sh | 3 -- dataplane-in-a-box/tfs-01-topo-nodes.json | 4 +- dataplane-in-a-box/tfs-02-topo-links.json | 36 +++++++++++------- .../tfs-03-dc1-2-dc2-l3svc.json | 24 ++++++++++++ dataplane-in-a-box/tfs-03-dc2dc-l2svc.json | 17 --------- dataplane-in-a-box/tfs-04-dc2dc-l3svc.json | 37 ------------------- 11 files changed, 67 insertions(+), 95 deletions(-) create mode 100755 dataplane-in-a-box/clab-cli-r1.sh create mode 100755 dataplane-in-a-box/clab-cli-r2.sh delete mode 100755 dataplane-in-a-box/clab-cli-wan1.sh delete mode 100755 dataplane-in-a-box/clab-cli-wan2.sh create mode 100644 dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json delete mode 100644 dataplane-in-a-box/tfs-03-dc2dc-l2svc.json delete mode 100644 dataplane-in-a-box/tfs-04-dc2dc-l3svc.json diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 4dd22dec3..c6da22be0 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -16,7 +16,7 @@ source dataplane-in-a-box/deploy_specs.sh ## Download and install ContainerLab ```bash -sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 +sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.6 ``` ## Download Arista cEOS image and create Docker image @@ -46,13 +46,13 @@ sudo rm -rf clab-arista/ .arista.clab.yml.bak ## Access cEOS Bash ```bash -docker exec -it clab-arista-wan1 bash +docker exec -it clab-arista-r1 bash ``` ## Access cEOS CLI ```bash -docker exec -it clab-arista-wan1 Cli -docker exec -it clab-arista-wan2 Cli +docker exec -it clab-arista-r1 Cli +docker exec -it clab-arista-r2 Cli ``` ## Configure ContainerLab clients diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index fa4957f19..a7766e326 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -31,37 +31,31 @@ topology: image: ghcr.io/hellt/network-multitool:latest nodes: - wan1: + r1: kind: arista_ceos mgmt-ipv4: 172.20.20.101 - ports: [6001:6030] - wan2: + r2: kind: arista_ceos mgmt-ipv4: 172.20.20.102 - ports: [6002:6030] dc1: kind: linux - mgmt-ipv4: 172.20.20.201 - ports: [2201:22] + mgmt-ipv4: 172.20.20.211 exec: - - ip link set address 00:c1:ab:00:00:01 dev eth1 + - ip link set address 00:c1:ab:00:01:01 dev eth1 - ip address add 192.168.1.10/24 dev eth1 - ip route add 192.168.2.0/24 via 192.168.1.1 dc2: kind: linux - mgmt-ipv4: 172.20.20.202 - ports: [2202:22] + mgmt-ipv4: 172.20.20.221 exec: - - ip link set address 00:c1:ab:00:00:02 dev eth1 + - ip link set address 00:c1:ab:00:02:01 dev eth1 - ip address add 192.168.2.10/24 dev eth1 - ip route add 192.168.1.0/24 via 192.168.2.1 links: - - endpoints: ["wan1:eth1", "wan2:eth1"] - - endpoints: ["wan1:eth2", "wan2:eth2"] - - endpoints: ["wan1:eth3", "wan2:eth3"] - - endpoints: ["wan1:eth10", "dc1:eth1"] - - endpoints: ["wan2:eth10", "dc2:eth1"] + - endpoints: ["r1:eth1", "r2:eth1"] + - endpoints: ["r1:eth10", "dc1:eth1"] + - endpoints: ["r2:eth10", "dc2:eth1"] diff --git a/dataplane-in-a-box/clab-cli-r1.sh b/dataplane-in-a-box/clab-cli-r1.sh new file mode 100755 index 000000000..a4799c6ce --- /dev/null +++ b/dataplane-in-a-box/clab-cli-r1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-r1 Cli diff --git a/dataplane-in-a-box/clab-cli-r2.sh b/dataplane-in-a-box/clab-cli-r2.sh new file mode 100755 index 000000000..39dce3938 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-r2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-r2 Cli diff --git a/dataplane-in-a-box/clab-cli-wan1.sh b/dataplane-in-a-box/clab-cli-wan1.sh deleted file mode 100755 index 4ae21bcb5..000000000 --- a/dataplane-in-a-box/clab-cli-wan1.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker exec -it clab-arista-wan1 Cli diff --git a/dataplane-in-a-box/clab-cli-wan2.sh b/dataplane-in-a-box/clab-cli-wan2.sh deleted file mode 100755 index c931ac940..000000000 --- a/dataplane-in-a-box/clab-cli-wan2.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker exec -it clab-arista-wan2 Cli diff --git a/dataplane-in-a-box/tfs-01-topo-nodes.json b/dataplane-in-a-box/tfs-01-topo-nodes.json index d0fafbfe8..7331664a1 100644 --- a/dataplane-in-a-box/tfs-01-topo-nodes.json +++ b/dataplane-in-a-box/tfs-01-topo-nodes.json @@ -27,7 +27,7 @@ ]} }, { - "device_id": {"device_uuid": {"uuid": "wan1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "r1"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, @@ -37,7 +37,7 @@ ]} }, { - "device_id": {"device_uuid": {"uuid": "wan2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "r2"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json index 78765a7d5..3e65f5b28 100644 --- a/dataplane-in-a-box/tfs-02-topo-links.json +++ b/dataplane-in-a-box/tfs-02-topo-links.json @@ -1,39 +1,47 @@ { "links": [ { - "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/1==wan2/ethernet-1/1"}}, + "link_id": {"link_uuid": {"uuid": "r1/Ethernet1==r2/Ethernet1"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} ] }, { - "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/2"}}, + "link_id": {"link_uuid": {"uuid": "r2/Ethernet1==r1/Ethernet1"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} ] }, + { - "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/3==wan2/ethernet-1/3"}}, + "link_id": {"link_uuid": {"uuid": "r1/Ethernet10==dc1/eth1"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}} + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} ] }, - { - "link_id": {"link_uuid": {"uuid": "dc1/eth1==wan1/ethernet-1/10"}}, + "link_id": {"link_uuid": {"uuid": "dc1/eth1==r1/Ethernet10"}}, "link_endpoint_ids": [ {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "r2/Ethernet10==dc2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} ] }, { - "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan2/ethernet-1/10"}}, + "link_id": {"link_uuid": {"uuid": "dc2/eth1==r2/Ethernet10"}}, "link_endpoint_ids": [ {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} ] } ] diff --git a/dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json b/dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json new file mode 100644 index 000000000..0e2dc5c67 --- /dev/null +++ b/dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json @@ -0,0 +1,24 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc1-2-dc2-l3svc"} + }, + "service_type": 1, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/device[dc1]/endpoint[eth1]/settings", "resource_value": { + "ipv4_address": "192.168.1.10", "ipv4_prefix": 24, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[dc2]/endpoint[eth1]/settings", "resource_value": { + "ipv4_address": "192.168.2.10", "ipv4_prefix": 24, "sub_interface_index": 0 + }}} + ]} + } + ] +} diff --git a/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json deleted file mode 100644 index 8d10e5f4b..000000000 --- a/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "services": [ - { - "service_id": { - "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l2svc"} - }, - "service_type": 2, - "service_status": {"service_status": 1}, - "service_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, - {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} - ], - "service_constraints": [], - "service_config": {"config_rules": []} - } - ] -} diff --git a/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json deleted file mode 100644 index b21cba0da..000000000 --- a/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "services": [ - { - "service_id": { - "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l3svc"} - }, - "service_type": 1, - "service_status": {"service_status": 1}, - "service_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, - {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} - ], - "service_constraints": [], - "service_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "/device[wan1]/settings", "resource_value": { - "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] - }}}, - {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/1]/settings", "resource_value": { - "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 - }}}, - {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/2]/settings", "resource_value": { - "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 - }}}, - - {"action": 1, "custom": {"resource_key": "/device[wan3]/settings", "resource_value": { - "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] - }}}, - {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/1]/settings", "resource_value": { - "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 - }}}, - {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/2]/settings", "resource_value": { - "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 - }}} - ]} - } - ] -} -- GitLab From ba1d60780ee4ac953ab318a9ff79189c6ba6a8f7 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 08:02:06 +0000 Subject: [PATCH 24/34] DataPlane-in-a-box: - Updated copyright headers --- dataplane-in-a-box/arista.clab.yml | 4 ++-- dataplane-in-a-box/clab-cli-dc1.sh | 13 +++++++++++++ dataplane-in-a-box/clab-cli-dc2.sh | 13 +++++++++++++ dataplane-in-a-box/clab-cli-r1.sh | 13 +++++++++++++ dataplane-in-a-box/clab-cli-r2.sh | 13 +++++++++++++ dataplane-in-a-box/clab-deploy.sh | 2 +- dataplane-in-a-box/clab-destroy.sh | 2 +- dataplane-in-a-box/clab-inspect.sh | 2 +- dataplane-in-a-box/deploy_specs.sh | 2 +- 9 files changed, 58 insertions(+), 6 deletions(-) diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index a7766e326..8bed10773 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -1,10 +1,10 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dataplane-in-a-box/clab-cli-dc1.sh b/dataplane-in-a-box/clab-cli-dc1.sh index 7d793f035..fc47fecdb 100755 --- a/dataplane-in-a-box/clab-cli-dc1.sh +++ b/dataplane-in-a-box/clab-cli-dc1.sh @@ -1,3 +1,16 @@ #!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker exec -it clab-arista-dc1 bash diff --git a/dataplane-in-a-box/clab-cli-dc2.sh b/dataplane-in-a-box/clab-cli-dc2.sh index 311d6dae5..0f308b532 100755 --- a/dataplane-in-a-box/clab-cli-dc2.sh +++ b/dataplane-in-a-box/clab-cli-dc2.sh @@ -1,3 +1,16 @@ #!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker exec -it clab-arista-dc2 bash diff --git a/dataplane-in-a-box/clab-cli-r1.sh b/dataplane-in-a-box/clab-cli-r1.sh index a4799c6ce..807ec0517 100755 --- a/dataplane-in-a-box/clab-cli-r1.sh +++ b/dataplane-in-a-box/clab-cli-r1.sh @@ -1,3 +1,16 @@ #!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker exec -it clab-arista-r1 Cli diff --git a/dataplane-in-a-box/clab-cli-r2.sh b/dataplane-in-a-box/clab-cli-r2.sh index 39dce3938..d9eea8932 100755 --- a/dataplane-in-a-box/clab-cli-r2.sh +++ b/dataplane-in-a-box/clab-cli-r2.sh @@ -1,3 +1,16 @@ #!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker exec -it clab-arista-r2 Cli diff --git a/dataplane-in-a-box/clab-deploy.sh b/dataplane-in-a-box/clab-deploy.sh index 2b8e49a07..b66480b9b 100755 --- a/dataplane-in-a-box/clab-deploy.sh +++ b/dataplane-in-a-box/clab-deploy.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/dataplane-in-a-box/clab-destroy.sh b/dataplane-in-a-box/clab-destroy.sh index 4030239dc..1cf83516a 100755 --- a/dataplane-in-a-box/clab-destroy.sh +++ b/dataplane-in-a-box/clab-destroy.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/dataplane-in-a-box/clab-inspect.sh b/dataplane-in-a-box/clab-inspect.sh index 02024ec47..7037602cd 100755 --- a/dataplane-in-a-box/clab-inspect.sh +++ b/dataplane-in-a-box/clab-inspect.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh index 93d5b2b82..b96437ca3 100755 --- a/dataplane-in-a-box/deploy_specs.sh +++ b/dataplane-in-a-box/deploy_specs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- GitLab From 950867f705e71521a2ec7b211b13d03267da4c51 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 08:14:37 +0000 Subject: [PATCH 25/34] DataPlane-in-a-box: - Removed unneeded files --- dataplane-in-a-box/example_config/wan1.conf | 37 ------------------- dataplane-in-a-box/example_config/wan2.conf | 37 ------------------- dataplane-in-a-box/links.txt | 8 ---- .../service/drivers/gnmi_openconfig/TODO.txt | 15 -------- 4 files changed, 97 deletions(-) delete mode 100644 dataplane-in-a-box/example_config/wan1.conf delete mode 100644 dataplane-in-a-box/example_config/wan2.conf delete mode 100644 dataplane-in-a-box/links.txt delete mode 100644 src/device/service/drivers/gnmi_openconfig/TODO.txt diff --git a/dataplane-in-a-box/example_config/wan1.conf b/dataplane-in-a-box/example_config/wan1.conf deleted file mode 100644 index fccb7b43f..000000000 --- a/dataplane-in-a-box/example_config/wan1.conf +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable -configure -ip routing - -interface Ethernet1 - no switchport - ip address 10.1.2.1/30 -exit - -interface Loopback0 - ip address 10.0.0.1/32 -exit - -interface Ethernet10 - no switchport - ip address 192.168.1.1/24 -exit - -router bgp 65001 - router-id 10.0.0.1 - neighbor 10.1.2.2 remote-as 65001 - network 192.168.1.0/24 -exit diff --git a/dataplane-in-a-box/example_config/wan2.conf b/dataplane-in-a-box/example_config/wan2.conf deleted file mode 100644 index 6edc0c405..000000000 --- a/dataplane-in-a-box/example_config/wan2.conf +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable -configure -ip routing - -interface Ethernet1 - no switchport - ip address 10.1.2.2/30 -exit - -interface Loopback0 - ip address 10.0.0.2/32 -exit - -interface Ethernet10 - no switchport - ip address 192.168.2.1/24 -exit - -router bgp 65001 - router-id 10.0.0.2 - neighbor 10.1.2.1 remote-as 65001 - network 192.168.2.0/24 -exit diff --git a/dataplane-in-a-box/links.txt b/dataplane-in-a-box/links.txt deleted file mode 100644 index a61ad5398..000000000 --- a/dataplane-in-a-box/links.txt +++ /dev/null @@ -1,8 +0,0 @@ -https://containerlab.dev/manual/multi-node/#exposing-services -https://containerlab.dev/manual/multi-node/#bridging -https://containerlab.dev/manual/kinds/bridge/ -https://containerlab.dev/lab-examples/ext-bridge/ - -https://containerlab.dev/manual/kinds/ceos/ -https://containerlab.dev/lab-examples/srl-ceos/#__tabbed_2_2 -https://github.com/srl-labs/containerlab/blob/main/lab-examples/srlceos01/srlceos01.clab.yml diff --git a/src/device/service/drivers/gnmi_openconfig/TODO.txt b/src/device/service/drivers/gnmi_openconfig/TODO.txt deleted file mode 100644 index ba8ff1c2c..000000000 --- a/src/device/service/drivers/gnmi_openconfig/TODO.txt +++ /dev/null @@ -1,15 +0,0 @@ -- update parse() @ InterfaceCounter.py -- update compose() @ NetworkInstance.py -- update compose() @ NetworkInstanceInterface.py -- implement parse() @ NetworkInstanceInterface.py -- update compose() @ NetworkInstanceStaticRoute.py -- implement parse() @ NetworkInstanceStaticRoute.py -- Fix MonitoringThread.py - - -there is an error removing static routes that makes unitary tests to crash -uncomment commented check_config_network_instance and validate - -- implement L2 VPN with BGP -- implement L3 VPN with BGP -- test static routes with ping -- GitLab From 3ff88bbabf660460ffd9e9d9a26205e2830b487e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 09:07:06 +0000 Subject: [PATCH 26/34] DataPlane-in-a-box: - Updated TFS descriptors - Updated TFS deploy specs --- dataplane-in-a-box/deploy_specs.sh | 2 +- ...1-topo-nodes.json => tfs-01-topology.json} | 46 ++++++++++++++++++ ...-dc2-l3svc.json => tfs-02-l3-service.json} | 0 dataplane-in-a-box/tfs-02-topo-links.json | 48 ------------------- 4 files changed, 47 insertions(+), 49 deletions(-) rename dataplane-in-a-box/{tfs-01-topo-nodes.json => tfs-01-topology.json} (56%) rename dataplane-in-a-box/{tfs-03-dc1-2-dc2-l3svc.json => tfs-02-l3-service.json} (100%) delete mode 100644 dataplane-in-a-box/tfs-02-topo-links.json diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh index b96437ca3..86f3d8711 100755 --- a/dataplane-in-a-box/deploy_specs.sh +++ b/dataplane-in-a-box/deploy_specs.sh @@ -24,7 +24,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring -#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate ZTP #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" diff --git a/dataplane-in-a-box/tfs-01-topo-nodes.json b/dataplane-in-a-box/tfs-01-topology.json similarity index 56% rename from dataplane-in-a-box/tfs-01-topo-nodes.json rename to dataplane-in-a-box/tfs-01-topology.json index 7331664a1..73e1bc224 100644 --- a/dataplane-in-a-box/tfs-01-topo-nodes.json +++ b/dataplane-in-a-box/tfs-01-topology.json @@ -46,5 +46,51 @@ }}} ]} } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "r1/Ethernet1==r2/Ethernet1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "r2/Ethernet1==r1/Ethernet1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "r1/Ethernet10==dc1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "dc1/eth1==r1/Ethernet10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "r2/Ethernet10==dc2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "dc2/eth1==r2/Ethernet10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} + ] + } ] } diff --git a/dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json b/dataplane-in-a-box/tfs-02-l3-service.json similarity index 100% rename from dataplane-in-a-box/tfs-03-dc1-2-dc2-l3svc.json rename to dataplane-in-a-box/tfs-02-l3-service.json diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json deleted file mode 100644 index 3e65f5b28..000000000 --- a/dataplane-in-a-box/tfs-02-topo-links.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "links": [ - { - "link_id": {"link_uuid": {"uuid": "r1/Ethernet1==r2/Ethernet1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, - {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "r2/Ethernet1==r1/Ethernet1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet1"}}, - {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "r1/Ethernet10==dc1/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, - {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "dc1/eth1==r1/Ethernet10"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "r1"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "r2/Ethernet10==dc2/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}}, - {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "dc2/eth1==r2/Ethernet10"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "r2"}}, "endpoint_uuid": {"uuid": "Ethernet10"}} - ] - } - ] -} -- GitLab From 2859baadeaf76d68a7d4363036f6245a03d86aa5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 13:52:16 +0000 Subject: [PATCH 27/34] DataPlane-in-a-box: - Updated TFS descriptors --- dataplane-in-a-box/tfs-01-topology.json | 36 +++++++++++++---------- dataplane-in-a-box/tfs-02-l3-service.json | 18 +++++++----- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/dataplane-in-a-box/tfs-01-topology.json b/dataplane-in-a-box/tfs-01-topology.json index 73e1bc224..6362584ba 100644 --- a/dataplane-in-a-box/tfs-01-topology.json +++ b/dataplane-in-a-box/tfs-01-topology.json @@ -7,41 +7,45 @@ ], "devices": [ { - "device_id": {"device_uuid": {"uuid": "dc1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc1"}}, "device_type": "emu-datacenter", + "device_drivers": ["DEVICEDRIVER_UNDEFINED"], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "dc2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc2"}}, "device_type": "emu-datacenter", + "device_drivers": ["DEVICEDRIVER_UNDEFINED"], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "r1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "r1"}}, "device_type": "packet-router", + "device_drivers": ["DEVICEDRIVER_GNMI_OPENCONFIG"], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "r2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "r2"}}, "device_type": "packet-router", + "device_drivers": ["DEVICEDRIVER_GNMI_OPENCONFIG"], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} diff --git a/dataplane-in-a-box/tfs-02-l3-service.json b/dataplane-in-a-box/tfs-02-l3-service.json index 0e2dc5c67..59938e71e 100644 --- a/dataplane-in-a-box/tfs-02-l3-service.json +++ b/dataplane-in-a-box/tfs-02-l3-service.json @@ -4,20 +4,22 @@ "service_id": { "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc1-2-dc2-l3svc"} }, - "service_type": 1, - "service_status": {"service_status": 1}, + "service_type": "SERVICETYPE_L3NM", + "service_status": {"service_status": "SERVICESTATUS_PLANNED"}, "service_endpoint_ids": [ {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} ], "service_constraints": [], "service_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "/device[dc1]/endpoint[eth1]/settings", "resource_value": { - "ipv4_address": "192.168.1.10", "ipv4_prefix": 24, "sub_interface_index": 0 - }}}, - {"action": 1, "custom": {"resource_key": "/device[dc2]/endpoint[eth1]/settings", "resource_value": { - "ipv4_address": "192.168.2.10", "ipv4_prefix": 24, "sub_interface_index": 0 - }}} + {"action": "CONFIGACTION_SET", "custom": { + "resource_key": "/device[dc1]/endpoint[eth1]/settings", + "resource_value": {"ipv4_address": "192.168.1.10", "ipv4_prefix": 24, "sub_interface_index": 0} + }}, + {"action": "CONFIGACTION_SET", "custom": { + "resource_key": "/device[dc2]/endpoint[eth1]/settings", + "resource_value": {"ipv4_address": "192.168.2.10", "ipv4_prefix": 24, "sub_interface_index": 0} + }} ]} } ] -- GitLab From d01724214b5e8b6c4516e5c8544afac1e2ec009a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 13:52:53 +0000 Subject: [PATCH 28/34] Monitoring component: - Enhanced logic to detect enabled endpoints and devices --- src/monitoring/service/EventTools.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/monitoring/service/EventTools.py b/src/monitoring/service/EventTools.py index 7820f11c8..2ad31c9cb 100644 --- a/src/monitoring/service/EventTools.py +++ b/src/monitoring/service/EventTools.py @@ -108,12 +108,15 @@ class EventsDeviceCollector: if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue if config_rule.WhichOneof('config_rule') != 'custom': continue str_resource_key = str(config_rule.custom.resource_key) - if not str_resource_key.startswith('/interface['): continue - json_resource_value = json.loads(config_rule.custom.resource_value) - if 'name' not in json_resource_value: continue - if 'enabled' not in json_resource_value: continue - if not json_resource_value['enabled']: continue - enabled_endpoint_names.add(json_resource_value['name']) + if str_resource_key.startswith('/interface[') or str_resource_key.startswith('/endpoints/endpoint['): + json_resource_value = json.loads(config_rule.custom.resource_value) + if 'name' not in json_resource_value: continue + if 'enabled' in json_resource_value: + if not json_resource_value['enabled']: continue + enabled_endpoint_names.add(json_resource_value['name']) + if 'oper-status' in json_resource_value: + if str(json_resource_value['oper-status']).upper() != 'UP': continue + enabled_endpoint_names.add(json_resource_value['name']) endpoints_monitored = self._device_endpoint_monitored.setdefault(device_uuid, dict()) for endpoint in device.device_endpoints: @@ -127,7 +130,10 @@ class EventsDeviceCollector: endpoint_was_monitored = endpoints_monitored.get(endpoint_uuid, False) endpoint_is_enabled = (endpoint_name_or_uuid in enabled_endpoint_names) - if not endpoint_was_monitored and endpoint_is_enabled: + if not endpoint_was_monitored and not endpoint_is_enabled: + # endpoint is idle, do nothing + pass + elif not endpoint_was_monitored and endpoint_is_enabled: # activate for value in endpoint.kpi_sample_types: if value == KPISAMPLETYPE_UNKNOWN: continue -- GitLab From 684339313211cade5b452d25c12bf4550ae48e12 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 14:46:45 +0000 Subject: [PATCH 29/34] PathComp component - Frontend: - Added special logic to manage pure-packet connectivity services without sub-services --- .../frontend/service/algorithms/_Algorithm.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/pathcomp/frontend/service/algorithms/_Algorithm.py b/src/pathcomp/frontend/service/algorithms/_Algorithm.py index 3ed2b13fb..3394d8df9 100644 --- a/src/pathcomp/frontend/service/algorithms/_Algorithm.py +++ b/src/pathcomp/frontend/service/algorithms/_Algorithm.py @@ -14,6 +14,7 @@ import json, logging, requests, uuid from typing import Dict, List, Optional, Tuple, Union +from common.DeviceTypes import DeviceTypeEnum from common.proto.context_pb2 import ( ConfigRule, Connection, Device, DeviceList, EndPointId, Link, LinkList, Service, ServiceStatusEnum, ServiceTypeEnum ) @@ -251,21 +252,37 @@ class _Algorithm: ] self.logger.debug('path_hops = {:s}'.format(str(path_hops))) - try: - _device_dict = {k:v[0] for k,v in self.device_dict.items()} - self.logger.debug('self.device_dict = {:s}'.format(str(_device_dict))) - connections = convert_explicit_path_hops_to_connections( - path_hops, self.device_dict, main_service_uuid, main_service_type) - self.logger.debug('EXTRAPOLATED connections = {:s}'.format(str(connections))) - except: # pylint: disable=bare-except - MSG = ' '.join([ - 'Unable to Extrapolate sub-services and sub-connections.', - 'Assuming single-service and single-connection.', - ]) - self.logger.exception(MSG) + device_types = {v[0]['device_type'] for k,v in self.device_dict.items()} + DEVICES_BASIC_CONNECTION = { + DeviceTypeEnum.DATACENTER.value, DeviceTypeEnum.EMULATED_DATACENTER.value, + DeviceTypeEnum.CLIENT.value, DeviceTypeEnum.EMULATED_CLIENT.value, + DeviceTypeEnum.PACKET_ROUTER.value, DeviceTypeEnum.EMULATED_PACKET_ROUTER.value, + } + self.logger.debug('device_types = {:s}'.format(str(device_types))) + self.logger.debug('DEVICES_BASIC_CONNECTION = {:s}'.format(str(DEVICES_BASIC_CONNECTION))) + is_basic_connection = device_types.issubset(DEVICES_BASIC_CONNECTION) + self.logger.debug('is_basic_connection = {:s}'.format(str(is_basic_connection))) + if is_basic_connection: + self.logger.info('Assuming basic connections...') connections = convert_explicit_path_hops_to_plain_connection( path_hops, main_service_uuid, main_service_type) self.logger.debug('BASIC connections = {:s}'.format(str(connections))) + else: + try: + _device_dict = {k:v[0] for k,v in self.device_dict.items()} + self.logger.debug('self.device_dict = {:s}'.format(str(_device_dict))) + connections = convert_explicit_path_hops_to_connections( + path_hops, self.device_dict, main_service_uuid, main_service_type) + self.logger.debug('EXTRAPOLATED connections = {:s}'.format(str(connections))) + except: # pylint: disable=bare-except + MSG = ' '.join([ + 'Unable to Extrapolate sub-services and sub-connections.', + 'Assuming single-service and single-connection.', + ]) + self.logger.exception(MSG) + connections = convert_explicit_path_hops_to_plain_connection( + path_hops, main_service_uuid, main_service_type) + self.logger.debug('BASIC connections = {:s}'.format(str(connections))) for connection in connections: service_uuid,service_type,path_hops,_ = connection -- GitLab From 4935a7087d7700dc1a3d2db7cf05042bed340c62 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 14:49:13 +0000 Subject: [PATCH 30/34] Service component: - Updated service handler management: now classifies devices by type, and instantiates a service handler per group of devices of the same type. --- .../service/task_scheduler/TaskExecutor.py | 62 +++++++++++-------- .../tasks/Task_ConnectionConfigure.py | 27 ++++++-- .../tasks/Task_ConnectionDeconfigure.py | 27 ++++++-- 3 files changed, 81 insertions(+), 35 deletions(-) diff --git a/src/service/service/task_scheduler/TaskExecutor.py b/src/service/service/task_scheduler/TaskExecutor.py index cd20faad2..08373230c 100644 --- a/src/service/service/task_scheduler/TaskExecutor.py +++ b/src/service/service/task_scheduler/TaskExecutor.py @@ -14,7 +14,8 @@ import json, logging from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from common.DeviceTypes import DeviceTypeEnum from common.method_wrappers.ServiceExceptions import NotFoundException from common.proto.context_pb2 import ( Connection, ConnectionId, Device, DeviceDriverEnum, DeviceId, Service, ServiceId, @@ -158,8 +159,8 @@ class TaskExecutor: def get_devices_from_connection( self, connection : Connection, exclude_managed_by_controller : bool = False - ) -> Dict[str, Device]: - devices = dict() + ) -> Dict[DeviceTypeEnum, Dict[str, Device]]: + devices : Dict[DeviceTypeEnum, Dict[str, Device]] = dict() for endpoint_id in connection.path_hops_endpoint_ids: device = self.get_device(endpoint_id.device_id) device_uuid = endpoint_id.device_id.device_uuid.uuid @@ -167,11 +168,14 @@ class TaskExecutor: controller = self.get_device_controller(device) if controller is None: - devices[device_uuid] = device + device_type = DeviceTypeEnum._value2member_map_[device.device_type] + devices.setdefault(device_type, dict())[device_uuid] = device else: if not exclude_managed_by_controller: - devices[device_uuid] = device - devices[controller.device_id.device_uuid.uuid] = controller + device_type = DeviceTypeEnum._value2member_map_[device.device_type] + devices.setdefault(device_type, dict())[device_uuid] = device + device_type = DeviceTypeEnum._value2member_map_[controller.device_type] + devices.setdefault(device_type, dict())[controller.device_id.device_uuid.uuid] = controller return devices # ----- Service-related methods ------------------------------------------------------------------------------------ @@ -198,25 +202,33 @@ class TaskExecutor: # ----- Service Handler Factory ------------------------------------------------------------------------------------ - def get_service_handler( + def get_service_handlers( self, connection : Connection, service : Service, **service_handler_settings - ) -> '_ServiceHandler': - connection_devices = self.get_devices_from_connection(connection, exclude_managed_by_controller=True) - try: - service_handler_class = get_service_handler_class( - self._service_handler_factory, service, connection_devices) - return service_handler_class(service, self, **service_handler_settings) - except (UnsatisfiedFilterException, UnsupportedFilterFieldException, UnsupportedFilterFieldValueException): - dict_connection_devices = { - cd_data.name : (cd_uuid, cd_data.name, { - (device_driver, DeviceDriverEnum.Name(device_driver)) - for device_driver in cd_data.device_drivers - }) - for cd_uuid,cd_data in connection_devices.items() - } - LOGGER.exception( - 'Unable to select service handler. service={:s} connection={:s} connection_devices={:s}'.format( + ) -> Dict[DeviceTypeEnum, Tuple['_ServiceHandler', Dict[str, Device]]]: + connection_device_types : Dict[DeviceTypeEnum, Dict[str, Device]] = self.get_devices_from_connection( + connection, exclude_managed_by_controller=True + ) + service_handlers : Dict[DeviceTypeEnum, Tuple['_ServiceHandler', Dict[str, Device]]] = dict() + for device_type, connection_devices in connection_device_types.items(): + try: + service_handler_class = get_service_handler_class( + self._service_handler_factory, service, connection_devices) + service_handler = service_handler_class(service, self, **service_handler_settings) + service_handlers[device_type] = (service_handler, connection_devices) + except ( + UnsatisfiedFilterException, UnsupportedFilterFieldException, + UnsupportedFilterFieldValueException + ): + dict_connection_devices = { + cd_data.name : (cd_uuid, cd_data.name, { + (device_driver, DeviceDriverEnum.Name(device_driver)) + for device_driver in cd_data.device_drivers + }) + for cd_uuid,cd_data in connection_devices.items() + } + MSG = 'Unable to select service handler. service={:s} connection={:s} connection_devices={:s}' + LOGGER.exception(MSG.format( grpc_message_to_json_string(service), grpc_message_to_json_string(connection), str(dict_connection_devices) - ) - ) + )) + return service_handlers diff --git a/src/service/service/task_scheduler/tasks/Task_ConnectionConfigure.py b/src/service/service/task_scheduler/tasks/Task_ConnectionConfigure.py index f6c543c1c..3f52f337a 100644 --- a/src/service/service/task_scheduler/tasks/Task_ConnectionConfigure.py +++ b/src/service/service/task_scheduler/tasks/Task_ConnectionConfigure.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import TYPE_CHECKING, Dict, Tuple +from common.DeviceTypes import DeviceTypeEnum from common.method_wrappers.ServiceExceptions import OperationFailedException -from common.proto.context_pb2 import ConnectionId +from common.proto.context_pb2 import ConnectionId, Device from common.tools.grpc.Tools import grpc_message_to_json_string from service.service.service_handler_api.Tools import check_errors_setendpoint from service.service.task_scheduler.TaskExecutor import TaskExecutor @@ -21,6 +23,9 @@ from service.service.tools.EndpointIdFormatters import endpointids_to_raw from service.service.tools.ObjectKeys import get_connection_key from ._Task import _Task +if TYPE_CHECKING: + from service.service.service_handler_api._ServiceHandler import _ServiceHandler + KEY_TEMPLATE = 'connection({connection_id:s}):configure' class Task_ConnectionConfigure(_Task): @@ -44,12 +49,24 @@ class Task_ConnectionConfigure(_Task): service = self._task_executor.get_service(connection.service_id) service_handler_settings = {} - service_handler = self._task_executor.get_service_handler(connection, service, **service_handler_settings) + service_handlers : Dict[DeviceTypeEnum, Tuple['_ServiceHandler', Dict[str, Device]]] = \ + self._task_executor.get_service_handlers(connection, service, **service_handler_settings) - endpointids_to_set = endpointids_to_raw(connection.path_hops_endpoint_ids) connection_uuid = connection.connection_id.connection_uuid.uuid - results_setendpoint = service_handler.SetEndpoint(endpointids_to_set, connection_uuid=connection_uuid) - errors = check_errors_setendpoint(endpointids_to_set, results_setendpoint) + endpointids_to_set = endpointids_to_raw(connection.path_hops_endpoint_ids) + + errors = list() + for _, (service_handler, connection_devices) in service_handlers.items(): + _endpointids_to_set = [ + (device_uuid, endpoint_uuid, topology_uuid) + for device_uuid, endpoint_uuid, topology_uuid in endpointids_to_set + if device_uuid in connection_devices + ] + results_setendpoint = service_handler.SetEndpoint( + _endpointids_to_set, connection_uuid=connection_uuid + ) + errors.extend(check_errors_setendpoint(endpointids_to_set, results_setendpoint)) + if len(errors) > 0: MSG = 'SetEndpoint for Connection({:s}) from Service({:s})' str_connection = grpc_message_to_json_string(connection) diff --git a/src/service/service/task_scheduler/tasks/Task_ConnectionDeconfigure.py b/src/service/service/task_scheduler/tasks/Task_ConnectionDeconfigure.py index 7b6b7951b..4ce774d20 100644 --- a/src/service/service/task_scheduler/tasks/Task_ConnectionDeconfigure.py +++ b/src/service/service/task_scheduler/tasks/Task_ConnectionDeconfigure.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import TYPE_CHECKING, Dict, Tuple +from common.DeviceTypes import DeviceTypeEnum from common.method_wrappers.ServiceExceptions import OperationFailedException -from common.proto.context_pb2 import ConnectionId +from common.proto.context_pb2 import ConnectionId, Device from common.tools.grpc.Tools import grpc_message_to_json_string from service.service.service_handler_api.Tools import check_errors_deleteendpoint from service.service.task_scheduler.TaskExecutor import TaskExecutor @@ -21,6 +23,9 @@ from service.service.tools.EndpointIdFormatters import endpointids_to_raw from service.service.tools.ObjectKeys import get_connection_key from ._Task import _Task +if TYPE_CHECKING: + from service.service.service_handler_api._ServiceHandler import _ServiceHandler + KEY_TEMPLATE = 'connection({connection_id:s}):deconfigure' class Task_ConnectionDeconfigure(_Task): @@ -44,12 +49,24 @@ class Task_ConnectionDeconfigure(_Task): service = self._task_executor.get_service(connection.service_id) service_handler_settings = {} - service_handler = self._task_executor.get_service_handler(connection, service, **service_handler_settings) + service_handlers : Dict[DeviceTypeEnum, Tuple['_ServiceHandler', Dict[str, Device]]] = \ + self._task_executor.get_service_handlers(connection, service, **service_handler_settings) - endpointids_to_delete = endpointids_to_raw(connection.path_hops_endpoint_ids) connection_uuid = connection.connection_id.connection_uuid.uuid - results_deleteendpoint = service_handler.DeleteEndpoint(endpointids_to_delete, connection_uuid=connection_uuid) - errors = check_errors_deleteendpoint(endpointids_to_delete, results_deleteendpoint) + endpointids_to_delete = endpointids_to_raw(connection.path_hops_endpoint_ids) + + errors = list() + for _, (service_handler, connection_devices) in service_handlers.items(): + _endpointids_to_delete = [ + (device_uuid, endpoint_uuid, topology_uuid) + for device_uuid, endpoint_uuid, topology_uuid in endpointids_to_delete + if device_uuid in connection_devices + ] + results_deleteendpoint = service_handler.DeleteEndpoint( + _endpointids_to_delete, connection_uuid=connection_uuid + ) + errors.extend(check_errors_deleteendpoint(endpointids_to_delete, results_deleteendpoint)) + if len(errors) > 0: MSG = 'DeleteEndpoint for Connection({:s}) from Service({:s})' str_connection = grpc_message_to_json_string(connection) -- GitLab From 9b65d30ac36f680a6ac9bf47169807dea7b7f79a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 14:50:30 +0000 Subject: [PATCH 31/34] Device component - gNMI OpenConfig driver: - Corrected interface parsing - Activated Monitoring Thread - Corrected paths and values in Monitoring Thread --- .../gnmi_openconfig/GnmiSessionHandler.py | 14 +++---- .../gnmi_openconfig/MonitoringThread.py | 18 ++++++--- .../gnmi_openconfig/handlers/Interface.py | 40 +++++++------------ 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 9c2c0abb3..f0ef6520d 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -25,7 +25,7 @@ from .tools.Channel import get_grpc_channel from .tools.Path import path_from_string, path_to_string #, compose_path from .tools.Subscriptions import Subscriptions from .tools.Value import decode_value #, value_exists -#from .MonitoringThread import MonitoringThread +from .MonitoringThread import MonitoringThread class GnmiSessionHandler: def __init__(self, address : str, port : int, settings : Dict, logger : logging.Logger) -> None: @@ -41,7 +41,7 @@ class GnmiSessionHandler: self._channel : Optional[grpc.Channel] = None self._stub : Optional[gNMIStub] = None self._yang_handler = YangHandler() - #self._monit_thread = None + self._monit_thread = None self._subscriptions = Subscriptions() self._in_subscriptions = queue.Queue() self._out_samples = queue.Queue() @@ -68,16 +68,16 @@ class GnmiSessionHandler: self._channel = get_grpc_channel(self._address, self._port, self._use_tls, self._logger) self._stub = gNMIStub(self._channel) check_capabilities(self._stub, self._username, self._password, timeout=120) - #self._monit_thread = MonitoringThread( - # self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) - #self._monit_thread.start() + self._monit_thread = MonitoringThread( + self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) + self._monit_thread.start() self._connected.set() def disconnect(self): if not self._connected.is_set(): return with self._lock: - #self._monit_thread.stop() - #self._monit_thread.join() + self._monit_thread.stop() + self._monit_thread.join() self._channel.close() self._connected.clear() diff --git a/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py b/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py index 8bf6704a8..c3668a86e 100644 --- a/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py +++ b/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py @@ -147,16 +147,22 @@ class MonitoringThread(threading.Thread): timestamp = timestamp_local for update_entry in update.update: str_path = path_to_string(update_entry.path) + if str_path.startswith('/interfaces/'): + # Add namespace, if missing + str_path_parts = str_path.split('/') + str_path_parts[1] = 'openconfig-interfaces:interfaces' + str_path = '/'.join(str_path_parts) #if str_path != '/system/name/host-name': continue #counter_name = update_entry.path[-1].name value_type = update_entry.val.WhichOneof('value') value = getattr(update_entry.val, value_type) - if re.match(r'^[0-9]+$', value) is not None: - value = int(value) - elif re.match(r'^[0-9]*\.[0-9]*$', value) is not None: - value = float(value) - else: - value = str(value) + if isinstance(value, str): + if re.match(r'^[0-9]+$', value) is not None: + value = int(value) + elif re.match(r'^[0-9]*\.[0-9]*$', value) is not None: + value = float(value) + else: + value = str(value) delta_sample = self._delta_sample_cache.get_delta(str_path, timestamp, value) if delta_sample is None: sample = (timestamp, str_path, value) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index a769692ba..aba6f4aac 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -149,8 +149,6 @@ class InterfaceHandler(_Handler): _subinterface['name'] = subinterface_state['name'] if 'enabled' in subinterface_state: _subinterface['enabled'] = subinterface_state['enabled'] - entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface_index) - entries.append((entry_subinterface_key, _subinterface)) if 'vlan' in subinterface: vlan = subinterface['vlan'] @@ -160,44 +158,36 @@ class InterfaceHandler(_Handler): if single_tagged is not None: single_tagged_config = single_tagged['config'] vlan_id = single_tagged_config['vlan-id'] - - _vlan = {'vlan_id': vlan_id} - entry_vlan_key = '{:s}/vlan[single:{:s}]'.format(entry_subinterface_key, vlan_id) - entries.append((entry_vlan_key, _vlan)) + _subinterface['vlan_id'] = vlan_id if len(vlan_match) > 0: raise Exception('Unsupported VLAN schema: {:s}'.format(str(vlan))) ipv4_addresses = subinterface.get('ipv4', {}).get('addresses', {}).get('address', []) + if len(ipv4_addresses) > 1: + raise Exception('Multiple IPv4 Addresses not supported: {:s}'.format(str(ipv4_addresses))) for ipv4_address in ipv4_addresses: LOGGER.debug('ipv4_address={:s}'.format(str(ipv4_address))) - - ipv4_address_ip = ipv4_address['ip'] + _subinterface['address_ip'] = ipv4_address['ip'] ipv4_address_state = ipv4_address.get('state', {}) - - _ipv4_address = {'ip': ipv4_address_ip} - if 'origin' in ipv4_address_state: - _ipv4_address['origin'] = ipv4_address_state['origin'] + #if 'origin' in ipv4_address_state: + # _subinterface['origin'] = ipv4_address_state['origin'] if 'prefix-length' in ipv4_address_state: - _ipv4_address['prefix'] = ipv4_address_state['prefix-length'] - - entry_ipv4_address_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, ipv4_address_ip) - entries.append((entry_ipv4_address_key, _ipv4_address)) + _subinterface['address_prefix'] = ipv4_address_state['prefix-length'] ipv6_addresses = subinterface.get('ipv6', {}).get('addresses', {}).get('address', []) + if len(ipv6_addresses) > 1: + raise Exception('Multiple IPv6 Addresses not supported: {:s}'.format(str(ipv6_addresses))) for ipv6_address in ipv6_addresses: LOGGER.debug('ipv6_address={:s}'.format(str(ipv6_address))) - - ipv6_address_ip = ipv6_address['ip'] + _subinterface['address_ipv6'] = ipv6_address['ip'] ipv6_address_state = ipv6_address.get('state', {}) - - _ipv6_address = {'ip': ipv6_address_ip} - if 'origin' in ipv6_address_state: - _ipv6_address['origin'] = ipv6_address_state['origin'] + #if 'origin' in ipv6_address_state: + # _subinterface['origin_ipv6'] = ipv6_address_state['origin'] if 'prefix-length' in ipv6_address_state: - _ipv6_address['prefix'] = ipv6_address_state['prefix-length'] + _subinterface['address_prefix_ipv6'] = ipv6_address_state['prefix-length'] - entry_ipv6_address_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, ipv6_address_ip) - entries.append((entry_ipv6_address_key, _ipv6_address)) + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface_index) + entries.append((entry_subinterface_key, _subinterface)) return entries -- GitLab From d231fa1673cf0d577f5d674bfe1e369cd1989d47 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 16:08:08 +0000 Subject: [PATCH 32/34] DataPlane-in-a-box: - Updated TFS descriptors --- dataplane-in-a-box/tfs-02-l3-service.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataplane-in-a-box/tfs-02-l3-service.json b/dataplane-in-a-box/tfs-02-l3-service.json index 59938e71e..98d5d4327 100644 --- a/dataplane-in-a-box/tfs-02-l3-service.json +++ b/dataplane-in-a-box/tfs-02-l3-service.json @@ -14,11 +14,11 @@ "service_config": {"config_rules": [ {"action": "CONFIGACTION_SET", "custom": { "resource_key": "/device[dc1]/endpoint[eth1]/settings", - "resource_value": {"ipv4_address": "192.168.1.10", "ipv4_prefix": 24, "sub_interface_index": 0} + "resource_value": {"address_ip": "192.168.1.10", "address_prefix": 24, "index": 0} }}, {"action": "CONFIGACTION_SET", "custom": { "resource_key": "/device[dc2]/endpoint[eth1]/settings", - "resource_value": {"ipv4_address": "192.168.2.10", "ipv4_prefix": 24, "sub_interface_index": 0} + "resource_value": {"address_ip": "192.168.2.10", "address_prefix": 24, "index": 0} }} ]} } -- GitLab From d41a3bc7b2a09a09102174a7b88d9f6b52cba14b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 16:11:13 +0000 Subject: [PATCH 33/34] Device component - gNMI OpenConfig driver: - Added missing NetworkInstanceProtocol handler - Corrected NetworkInstanceStaticRoute handler - Corrected division-by-zero bug in DeltaSampleCache - Reduced levels of log messages --- .../gnmi_openconfig/DeltaSampleCache.py | 16 ++-- .../gnmi_openconfig/MonitoringThread.py | 8 +- .../handlers/NetworkInstanceProtocol.py | 80 +++++++++++++++++++ .../handlers/NetworkInstanceStaticRoute.py | 33 ++++---- .../gnmi_openconfig/handlers/__init__.py | 4 + 5 files changed, 119 insertions(+), 22 deletions(-) create mode 100644 src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceProtocol.py diff --git a/src/device/service/drivers/gnmi_openconfig/DeltaSampleCache.py b/src/device/service/drivers/gnmi_openconfig/DeltaSampleCache.py index daf04be5a..140efe840 100644 --- a/src/device/service/drivers/gnmi_openconfig/DeltaSampleCache.py +++ b/src/device/service/drivers/gnmi_openconfig/DeltaSampleCache.py @@ -13,13 +13,15 @@ # limitations under the License. import copy -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union class DeltaSampleCache: def __init__(self) -> None: self._previous_samples : Dict[str, Tuple[float, Union[int, float]]] = dict() - def get_delta(self, path : str, current_timestamp : float, current_value : Any) -> None: + def get_delta( + self, path : str, current_timestamp : float, current_value : Any + ) -> Optional[Tuple[float, Optional[Any]]]: previous_sample = copy.deepcopy(self._previous_samples.get(path)) self._previous_samples[path] = current_timestamp, current_value @@ -30,6 +32,10 @@ class DeltaSampleCache: delta_value = max(0, current_value - previous_value) delay = current_timestamp - previous_timestamp - delta_sample = current_timestamp, delta_value / delay - - return delta_sample + if delay < 1.e-12: + # return a special value meaning, at that timestamp, + # computed value is not a number, e.g., division by zero + # also, recover previuos samples to do not miss any packet/byte + self._previous_samples[path] = previous_sample + return current_timestamp, None + return current_timestamp, delta_value / delay diff --git a/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py b/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py index c3668a86e..1910c1671 100644 --- a/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py +++ b/src/device/service/drivers/gnmi_openconfig/MonitoringThread.py @@ -134,7 +134,7 @@ class MonitoringThread(threading.Thread): self._response_iterator = self._stub.Subscribe(request_iterator, metadata=metadata, timeout=timeout) for subscribe_response in self._response_iterator: str_subscribe_response = grpc_message_to_json_string(subscribe_response) - self._logger.warning('[run] subscribe_response={:s}'.format(str_subscribe_response)) + self._logger.debug('[run] subscribe_response={:s}'.format(str_subscribe_response)) update = subscribe_response.update timestamp_device = float(update.timestamp) / 1.e9 timestamp_local = datetime.timestamp(datetime.utcnow()) @@ -168,8 +168,10 @@ class MonitoringThread(threading.Thread): sample = (timestamp, str_path, value) else: sample = (delta_sample[0], str_path, delta_sample[1]) - self._logger.warning('[run] sample={:s}'.format(str(sample))) - self._out_samples.put_nowait(sample) + self._logger.debug('[run] sample={:s}'.format(str(sample))) + if sample[2] is not None: + # Skip not-a-number (e.g., division by zero) samples + self._out_samples.put_nowait(sample) except grpc.RpcError as e: if e.code() != grpc.StatusCode.CANCELLED: raise # pylint: disable=no-member if e.details() != 'Locally cancelled by application!': raise # pylint: disable=no-member diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceProtocol.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceProtocol.py new file mode 100644 index 000000000..f45646192 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceProtocol.py @@ -0,0 +1,80 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, libyang, logging +from typing import Any, Dict, List, Tuple +from ._Handler import _Handler +from .Tools import get_str +from .YangHandler import YangHandler + +LOGGER = logging.getLogger(__name__) + +class NetworkInstanceProtocolHandler(_Handler): + def get_resource_key(self) -> str: return '/network_instance/protocols' + def get_path(self) -> str: + return '/openconfig-network-instance:network-instances/network-instance/protocols/protocol' + + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + identifier = get_str(resource_value, 'identifier') # 'STATIC' + proto_name = get_str(resource_value, 'protocol_name') # 'STATIC' + + if ':' not in identifier: + identifier = 'openconfig-policy-types:{:s}'.format(identifier) + PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' + str_path = PATH_TMPL.format(ni_name, identifier, proto_name) + + if delete: + str_data = json.dumps({}) + return str_path, str_data + + #str_data = json.dumps({ + # 'identifier': identifier, 'name': name, + # 'config': {'identifier': identifier, 'name': name, 'enabled': True}, + # 'static_routes': {'static': [{ + # 'prefix': prefix, + # 'config': {'prefix': prefix}, + # 'next_hops': { + # 'next-hop': [{ + # 'index': next_hop_index, + # 'config': {'index': next_hop_index, 'next_hop': next_hop} + # }] + # } + # }]} + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_prs : libyang.DContainer = yang_ni.create_path('protocols') + yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, proto_name) + yang_ni_pr : libyang.DContainer = yang_ni_prs.create_path(yang_ni_pr_path) + yang_ni_pr.create_path('config/identifier', identifier) + yang_ni_pr.create_path('config/name', proto_name) + yang_ni_pr.create_path('config/enabled', True ) + + str_data = yang_ni_pr.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:protocol'][0] + str_data = json.dumps(json_data) + return str_path, str_data + + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) + response = [] + return response diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index cc561f37d..ad1ef8b70 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -28,24 +28,29 @@ class NetworkInstanceStaticRouteHandler(_Handler): def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - ni_name = get_str(resource_value, 'name' ) # test-svc - prefix = get_str(resource_value, 'prefix') # '172.0.1.0/24' + ni_name = get_str(resource_value, 'name' ) # test-svc + identifier = get_str(resource_value, 'identifier') # 'STATIC' + proto_name = get_str(resource_value, 'protocol_name') # 'STATIC' + prefix = get_str(resource_value, 'prefix') # '172.0.1.0/24' + + if ':' not in identifier: + identifier = 'openconfig-policy-types:{:s}'.format(identifier) - identifier = 'openconfig-policy-types:STATIC' - name = 'static' if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols' PATH_TMPL += '/protocol[identifier={:s}][name={:s}]/static-routes/static[prefix={:s}]' - str_path = PATH_TMPL.format(ni_name, identifier, name, prefix) + str_path = PATH_TMPL.format(ni_name, identifier, proto_name, prefix) str_data = json.dumps({}) return str_path, str_data - next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' - metric = get_int(resource_value, 'metric' ) # 20 - next_hop_index = get_str(resource_value, 'next_hop_index') # AUTO_1_172-0-0-1 + next_hop = get_str(resource_value, 'next_hop') # '172.0.0.1' + metric = get_int(resource_value, 'metric' ) # 20 + index = get_str(resource_value, 'index' ) # AUTO_1_172-0-0-1 + if index is None: + index = 'AUTO_{:d}_{:s}'.format(metric, next_hop) PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' - str_path = PATH_TMPL.format(ni_name, identifier, name) + str_path = PATH_TMPL.format(ni_name, identifier, proto_name) #str_data = json.dumps({ # 'identifier': identifier, 'name': name, # 'config': {'identifier': identifier, 'name': name, 'enabled': True}, @@ -64,10 +69,10 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) yang_ni_prs : libyang.DContainer = yang_ni.create_path('protocols') - yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, name) + yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, proto_name) yang_ni_pr : libyang.DContainer = yang_ni_prs.create_path(yang_ni_pr_path) yang_ni_pr.create_path('config/identifier', identifier) - yang_ni_pr.create_path('config/name', name ) + yang_ni_pr.create_path('config/name', proto_name) yang_ni_pr.create_path('config/enabled', True ) yang_ni_pr_srs : libyang.DContainer = yang_ni_pr.create_path('static-routes') @@ -76,11 +81,11 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_ni_pr_sr.create_path('config/prefix', prefix) yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') - yang_ni_pr_sr_nh_path = 'next-hop[index="{:s}"]'.format(next_hop_index) + yang_ni_pr_sr_nh_path = 'next-hop[index="{:s}"]'.format(index) yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) - yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) + yang_ni_pr_sr_nh.create_path('config/index', index) yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) - yang_ni_pr_sr_nh.create_path('config/metric', metric) + yang_ni_pr_sr_nh.create_path('config/metric', metric) str_data = yang_ni_pr.print_mem('json') LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 0b2b95f44..b36313bb2 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -21,6 +21,7 @@ from .Interface import InterfaceHandler from .InterfaceCounter import InterfaceCounterHandler from .NetworkInstance import NetworkInstanceHandler from .NetworkInstanceInterface import NetworkInstanceInterfaceHandler +from .NetworkInstanceProtocol import NetworkInstanceProtocolHandler from .NetworkInstanceStaticRoute import NetworkInstanceStaticRouteHandler from .Tools import get_schema from .YangHandler import YangHandler @@ -32,6 +33,7 @@ ifaceh = InterfaceHandler() ifctrh = InterfaceCounterHandler() nih = NetworkInstanceHandler() niifh = NetworkInstanceInterfaceHandler() +niph = NetworkInstanceProtocolHandler() nisrh = NetworkInstanceStaticRouteHandler() ALL_RESOURCE_KEYS = [ @@ -59,6 +61,7 @@ RESOURCE_KEY_TO_HANDLER = { ifctrh.get_resource_key() : ifctrh, nih.get_resource_key() : nih, niifh.get_resource_key() : niifh, + niph.get_resource_key() : niph, nisrh.get_resource_key() : nisrh, } @@ -68,6 +71,7 @@ PATH_TO_HANDLER = { ifctrh.get_path() : ifctrh, nih.get_path() : nih, niifh.get_path() : niifh, + niph.get_path() : niph, nisrh.get_path() : nisrh, } -- GitLab From 4881a30f55d1a6cb85a4071a8696442df0c9cb18 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 13 May 2024 16:15:17 +0000 Subject: [PATCH 34/34] Service component - L3NM gNMI OpenConfig Service Handler: - ConfigRuleComposer: Added ignore of management interfaces - ConfigRuleComposer: Corrected endpoint attribute names - ConfigRuleComposer: Added alias mappings from device/endpoint name to their UUID to prevent split-brain conditions - L3NMGnmiOpenConfigServiceHandler: implemented correct path data loading --- .../ConfigRuleComposer.py | 68 ++++++++++++++----- .../L3NMGnmiOpenConfigServiceHandler.py | 15 +++- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py index 42747a1ae..343c37f5d 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/ConfigRuleComposer.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, netaddr, re +import json, logging, netaddr, re from typing import Dict, List, Optional, Set, Tuple from common.DeviceTypes import DeviceTypeEnum from common.proto.context_pb2 import ConfigActionEnum, Device, EndPoint, Service from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set from service.service.service_handler_api.AnyTreeTools import TreeNode +LOGGER = logging.getLogger(__name__) + NETWORK_INSTANCE = 'teraflowsdn' -RE_IF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') -RE_SR = re.compile(r'^\/network_instance\[([^\]]+)\]\/protocols\[STATIC\]/route\[([^\:]+)\:([^\]]+)\]$') +RE_IF = re.compile(r'^\/interface\[([^\]]+)\]$') +RE_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') +RE_SR = re.compile(r'^\/network_instance\[([^\]]+)\]\/protocols\[STATIC\]/route\[([^\:]+)\:([^\]]+)\]$') def _interface( interface : str, if_type : Optional[str] = 'l3ipvlan', index : int = 0, vlan_id : Optional[int] = None, @@ -82,9 +85,9 @@ class EndpointComposer: self.objekt = endpoint_obj if settings is None: return json_settings : Dict = settings.value - self.ipv4_address = json_settings['ipv4_address'] - self.ipv4_prefix_len = json_settings['ipv4_prefix_len'] - self.sub_interface_index = json_settings['sub_interface_index'] + self.ipv4_address = json_settings['address_ip'] + self.ipv4_prefix_len = json_settings['address_prefix'] + self.sub_interface_index = json_settings.get('index', 0) def get_config_rules(self, network_instance_name : str, delete : bool = False) -> List[Dict]: if self.ipv4_address is None: return [] @@ -102,20 +105,25 @@ class EndpointComposer: def dump(self) -> Dict: return { - 'sub_interface_index' : self.sub_interface_index, - 'ipv4_address' : self.ipv4_address, - 'ipv4_prefix_len' : self.ipv4_prefix_len, + 'index' : self.sub_interface_index, + 'address_ip' : self.ipv4_address, + 'address_prefix': self.ipv4_prefix_len, } class DeviceComposer: def __init__(self, device_uuid : str) -> None: self.uuid = device_uuid self.objekt : Optional[Device] = None - self.endpoints : Dict[str, EndpointComposer] = dict() + self.aliases : Dict[str, str] = dict() # endpoint_name => endpoint_uuid + self.endpoints : Dict[str, EndpointComposer] = dict() # endpoint_uuid => EndpointComposer self.connected : Set[str] = set() self.static_routes : Dict[str, Dict[int, str]] = dict() # {prefix => {metric => next_hop}} - + + def set_endpoint_alias(self, endpoint_name : str, endpoint_uuid : str) -> None: + self.aliases[endpoint_name] = endpoint_uuid + def get_endpoint(self, endpoint_uuid : str) -> EndpointComposer: + endpoint_uuid = self.aliases.get(endpoint_uuid, endpoint_uuid) if endpoint_uuid not in self.endpoints: self.endpoints[endpoint_uuid] = EndpointComposer(endpoint_uuid) return self.endpoints[endpoint_uuid] @@ -123,17 +131,36 @@ class DeviceComposer: def configure(self, device_obj : Device, settings : Optional[TreeNode]) -> None: self.objekt = device_obj for endpoint_obj in device_obj.device_endpoints: + endpoint_uuid = endpoint_obj.endpoint_id.endpoint_uuid.uuid + self.set_endpoint_alias(endpoint_obj.name, endpoint_uuid) self.get_endpoint(endpoint_obj.name).configure(endpoint_obj, None) + # Find management interfaces + mgmt_ifaces = set() for config_rule in device_obj.device_config.config_rules: if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue if config_rule.WhichOneof('config_rule') != 'custom': continue config_rule_custom = config_rule.custom - match = RE_IF.match(config_rule_custom.resource_key) + if match is None: continue + if_name = match.groups()[0] + resource_value = json.loads(config_rule_custom.resource_value) + management = resource_value.get('management', False) + if management: mgmt_ifaces.add(if_name) + + # Find data plane interfaces + for config_rule in device_obj.device_config.config_rules: + if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue + if config_rule.WhichOneof('config_rule') != 'custom': continue + config_rule_custom = config_rule.custom + + match = RE_SUBIF.match(config_rule_custom.resource_key) if match is not None: if_name, subif_index = match.groups() + if if_name in mgmt_ifaces: continue resource_value = json.loads(config_rule_custom.resource_value) + if 'address_ip' not in resource_value: continue + if 'address_prefix' not in resource_value: continue ipv4_network = str(resource_value['address_ip']) ipv4_prefix_len = int(resource_value['address_prefix']) endpoint = self.get_endpoint(if_name) @@ -197,19 +224,24 @@ class DeviceComposer: class ConfigRuleComposer: def __init__(self) -> None: self.objekt : Optional[Service] = None - self.devices : Dict[str, DeviceComposer] = dict() + self.aliases : Dict[str, str] = dict() # device_name => device_uuid + self.devices : Dict[str, DeviceComposer] = dict() # device_uuid => DeviceComposer - def configure(self, service_obj : Service, settings : Optional[TreeNode]) -> None: - self.objekt = service_obj - if settings is None: return - #json_settings : Dict = settings.value - # For future use + def set_device_alias(self, device_name : str, device_uuid : str) -> None: + self.aliases[device_name] = device_uuid def get_device(self, device_uuid : str) -> DeviceComposer: + device_uuid = self.aliases.get(device_uuid, device_uuid) if device_uuid not in self.devices: self.devices[device_uuid] = DeviceComposer(device_uuid) return self.devices[device_uuid] + def configure(self, service_obj : Service, settings : Optional[TreeNode]) -> None: + self.objekt = service_obj + if settings is None: return + #json_settings : Dict = settings.value + # For future use + def get_config_rules( self, network_instance_name : str = NETWORK_INSTANCE, delete : bool = False ) -> Dict[str, List[Dict]]: diff --git a/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py index 9142c9d1e..88bb5655b 100644 --- a/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_gnmi_openconfig/L3NMGnmiOpenConfigServiceHandler.py @@ -15,13 +15,15 @@ import json, logging from typing import Any, Dict, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method -from common.proto.context_pb2 import ConfigRule, DeviceId, Service +from common.proto.context_pb2 import ConfigRule, ConnectionId, DeviceId, Service +from common.tools.object_factory.Connection import json_connection_id from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type from service.service.service_handler_api._ServiceHandler import _ServiceHandler from service.service.service_handler_api.SettingsHandler import SettingsHandler from service.service.service_handler_api.Tools import get_device_endpoint_uuids, get_endpoint_matching from service.service.task_scheduler.TaskExecutor import TaskExecutor +from service.service.tools.EndpointIdFormatters import endpointids_to_raw from .ConfigRuleComposer import ConfigRuleComposer from .StaticRouteGenerator import StaticRouteGenerator @@ -51,17 +53,20 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) device_settings = self.__settings_handler.get_device_settings(device_obj) + self.__config_rule_composer.set_device_alias(device_obj.name, device_uuid) _device = self.__config_rule_composer.get_device(device_obj.name) _device.configure(device_obj, device_settings) endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) + _device.set_endpoint_alias(endpoint_obj.name, endpoint_uuid) _endpoint = _device.get_endpoint(endpoint_obj.name) _endpoint.configure(endpoint_obj, endpoint_settings) self.__endpoint_map[(device_uuid, endpoint_uuid)] = (device_obj.name, endpoint_obj.name) self.__static_route_generator.compose(endpoints) + LOGGER.debug('config_rule_composer = {:s}'.format(json.dumps(self.__config_rule_composer.dump()))) def _do_configurations( self, config_rules_per_device : Dict[str, List[Dict]], endpoints : List[Tuple[str, str, Optional[str]]], @@ -99,7 +104,9 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] #service_uuid = self.__service.service_id.service_uuid.uuid - self._compose_config_rules(endpoints) + connection = self.__task_executor.get_connection(ConnectionId(**json_connection_id(connection_uuid))) + connection_endpoint_ids = endpointids_to_raw(connection.path_hops_endpoint_ids) + self._compose_config_rules(connection_endpoint_ids) #network_instance_name = service_uuid.split('-')[0] #config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=False) config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=False) @@ -115,7 +122,9 @@ class L3NMGnmiOpenConfigServiceHandler(_ServiceHandler): chk_type('endpoints', endpoints, list) if len(endpoints) == 0: return [] #service_uuid = self.__service.service_id.service_uuid.uuid - self._compose_config_rules(endpoints) + connection = self.__task_executor.get_connection(ConnectionId(**json_connection_id(connection_uuid))) + connection_endpoint_ids = endpointids_to_raw(connection.path_hops_endpoint_ids) + self._compose_config_rules(connection_endpoint_ids) #network_instance_name = service_uuid.split('-')[0] #config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=True) config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=True) -- GitLab