From 3e1df0378bc50c1d67a1c27674f810084bca0bc8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 18:59:10 +0000 Subject: [PATCH 001/985] Device component - gNMI OpenConfig driver: - Updated Component Handler - Updated Interface Handler - Updated InterfaceCounter Handler - Partial update of NetworkInstance Handler - Updated Tools - Added scripts to collect OpenConfig data model and build yang model bindings - Minor cosmetic changes - Updated test_gnmi.py and test_gnmi.sh script - Updated requirements.in --- src/device/requirements.in | 3 +- .../gnmi_openconfig/01-clone-yang-models.sh | 27 +++ .../gnmi_openconfig/02-build-yang-bindings.sh | 106 +++++++++ .../gnmi_openconfig/GnmiSessionHandler.py | 4 +- .../gnmi_openconfig/handlers/Component.py | 42 ++-- .../gnmi_openconfig/handlers/Interface.py | 223 +++++------------- .../handlers/InterfaceCounter.py | 84 +++---- .../handlers/NetworkInstance.py | 9 +- .../drivers/gnmi_openconfig/handlers/Tools.py | 22 +- .../gnmi_openconfig/handlers/__init__.py | 11 +- .../drivers/gnmi_openconfig/tools/Path.py | 6 +- .../drivers/gnmi_openconfig/tools/Value.py | 37 ++- src/device/tests/test_gnmi.py | 50 ++-- test_gnmi.sh | 17 ++ 14 files changed, 366 insertions(+), 275 deletions(-) create mode 100755 src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh create mode 100755 src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh create mode 100755 test_gnmi.sh diff --git a/src/device/requirements.in b/src/device/requirements.in index ece761571..d8a33455e 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -15,6 +15,7 @@ anytree==2.8.0 APScheduler==3.10.1 +bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 Jinja2==3.0.3 @@ -32,7 +33,7 @@ tabulate ipaddress macaddress yattag -pyang +pyang==2.6.* git+https://github.com/robshakir/pyangbind.git websockets==10.4 diff --git a/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh new file mode 100755 index 000000000..fe852f0e1 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig + +rm -rf ${GIT_BASE_PATH} + +OC_PUBLIC_PATH=${GIT_BASE_PATH}/public +mkdir -p ${OC_PUBLIC_PATH} +git clone https://github.com/openconfig/public.git ${OC_PUBLIC_PATH} + +#OC_HERCULES_PATH=${GIT_BASE_PATH}/hercules +#mkdir -p ${OC_HERCULES_PATH} +#git clone https://github.com/openconfig/hercules.git ${OC_HERCULES_PATH} diff --git a/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh new file mode 100755 index 000000000..ed4cf263f --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig +OC_PUBLIC_MODELS_PATH=${GIT_BASE_PATH}/public/release/models +IETF_MODELS_PATH=${GIT_BASE_PATH}/public/third_party/ietf +#OC_HERCULES_MODELS_PATH=${GIT_BASE_PATH}/hercules/yang + +OUT_FOLDER=openconfig +OUT_PATH=${BASE_PATH}/handlers +cd ${OUT_PATH} +export PYBINDPLUGIN=`/usr/bin/env python -c 'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'` + +# -p ${OC_HERCULES_MODELS_PATH}/ +# --split-class-dir openconfig_hercules +pyang --plugindir $PYBINDPLUGIN -p ${OC_PUBLIC_MODELS_PATH}/ -f pybind --split-class-dir ${OUT_FOLDER} \ + ${IETF_MODELS_PATH}/iana-if-type.yang \ + ${IETF_MODELS_PATH}/ietf-interfaces.yang \ + ${IETF_MODELS_PATH}/ietf-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv4-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv6-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match.yang \ + ${OC_PUBLIC_MODELS_PATH}/defined-sets/openconfig-defined-sets.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-aggregate.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ethernet.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ip.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/mpls/openconfig-mpls-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/openconfig-extensions.yang \ + ${OC_PUBLIC_MODELS_PATH}/optical-transport/openconfig-transport-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-common.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-controller-card.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-cpu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-ext.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fabric.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fan.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-healthz.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-integrated-circuit.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-linecard.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-pipeline-counters.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-port.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-psu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-software.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-transceiver.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-elements.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-mem-mgmt.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos.yang \ + ${OC_PUBLIC_MODELS_PATH}/system/openconfig-alarm-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-inet-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l2.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l3.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-errors.yang \ + + +openconfig-aft +openconfig-bgp +openconfig-igmp +openconfig-isis +openconfig-local-routing +openconfig-mpls +openconfig-ospfv2 +openconfig-pcep +openconfig-pim +openconfig-policy-forwarding +openconfig-policy-types +openconfig-routing-policy +openconfig-segment-routing + + + + +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-interfaces.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-chassis.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-linecard.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-node.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-port.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-qos.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules.yang \ diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 04dae4f5f..6f80ee82f 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -132,8 +132,8 @@ class GnmiSessionHandler: #resource_key_tuple[2] = True results.extend(parse(str_path, value)) except Exception as e: # pylint: disable=broad-except - MSG = 'Exception processing notification {:s}' - self._logger.exception(MSG.format(grpc_message_to_json_string(notification))) + MSG = 'Exception processing update {:s}' + self._logger.exception(MSG.format(grpc_message_to_json_string(update))) results.append((str_path, e)) # if validation fails, store the exception #_results = sorted(results.items(), key=lambda x: x[1][0]) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 0b3c1f970..cddf40d56 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,45 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging #, json +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +#pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' - def get_path(self) -> str: return '/components/component' + def get_path(self) -> str: return '/openconfig-platform:components' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_component_list : List[Dict] = json_data.get('component', []) - response = [] - for json_component in json_component_list: - #LOGGER.info('json_component = {:s}'.format(json.dumps(json_component))) - endpoint = {} + oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') + #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) - component_type = json_component.get('state', {}).get('type') - if component_type is None: continue - component_type = component_type.replace('oc-platform-types:', '') - component_type = component_type.replace('openconfig-platform-types:', '') - if component_type not in {'PORT'}: continue - endpoint['type'] = '-' + entries = [] + for component_key, oc_component in oc_components.component.items(): + #LOGGER.info('component_key={:s} oc_component={:s}'.format( + # component_key, pybindJSON.dumps(oc_component, mode='ietf') + #)) - #LOGGER.info('PORT json_component = {:s}'.format(json.dumps(json_component))) + component_name = oc_component.config.name - component_name = json_component.get('name') - if component_name is None: continue + component_type = oc_component.state.type + component_type = component_type.split(':')[-1] + if component_type not in {'PORT'}: continue # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. interface_name = component_name.lower().replace('-port', '') - endpoint['uuid'] = interface_name + endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED : PATH_IF_CTR.format(interface_name, 'in-octets' ), KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED : PATH_IF_CTR.format(interface_name, 'out-octets'), @@ -59,5 +59,7 @@ class ComponentHandler(_Handler): } if len(endpoint) == 0: continue - response.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - return response + + entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 20f79b3c2..77310d51d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -13,15 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' - def get_path(self) -> str: return '/interfaces/interface' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: if_name = str (resource_value['name' ]) # ethernet-1/1 @@ -63,186 +64,88 @@ class InterfaceHandler(_Handler): def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - #LOGGER.info('json_interface = {:s}'.format(json.dumps(json_interface))) + entries = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + #)) interface = {} + interface['name'] = oc_interface.config.name - interface_name = json_interface.get('name') - if interface_name is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue - interface['name'] = interface_name - - CONFIG_FIELDS = ('config', 'openconfig-interface:config', 'oci:config') - json_config : Dict = dict_get_first(json_interface, CONFIG_FIELDS, default={}) - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state : Dict = dict_get_first(json_interface, STATE_FIELDS, default={}) - - interface_type = json_config.get('type') - if interface_type is None: interface_type = json_state.get('type') - if interface_type is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue + interface_type = oc_interface.config.type interface_type = interface_type.replace('ianaift:', '') interface_type = interface_type.replace('iana-if-type:', '') interface['type'] = interface_type - interface_mtu = json_config.get('mtu') - if interface_mtu is None: interface_mtu = json_state.get('mtu') - if interface_mtu is not None: interface['mtu'] = int(interface_mtu) - - interface_enabled = json_config.get('enabled') - if interface_enabled is None: interface_enabled = json_state.get('enabled') - interface['enabled'] = False if interface_enabled is None else bool(interface_enabled) - - interface_management = json_config.get('management') - if interface_management is None: interface_management = json_state.get('management') - interface['management'] = False if interface_management is None else bool(interface_management) + interface['mtu' ] = oc_interface.config.mtu + interface['enabled' ] = oc_interface.config.enabled + interface['description' ] = oc_interface.config.description + interface['admin-status'] = oc_interface.state.admin_status + interface['oper-status' ] = oc_interface.state.oper_status + interface['management' ] = oc_interface.state.management - interface_descr = json_interface.get('config', {}).get('description') - if interface_descr is not None: interface['description'] = interface_descr + entry_interface_key = '/interface[{:s}]'.format(interface['name']) + entries.append((entry_interface_key, interface)) - json_subinterfaces = json_interface.get('subinterfaces', {}) - json_subinterface_list : List[Dict] = json_subinterfaces.get('subinterface', []) - - for json_subinterface in json_subinterface_list: - #LOGGER.info('json_subinterface = {:s}'.format(json.dumps(json_subinterface))) + for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): + #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( + # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') + #)) subinterface = {} + subinterface['index' ] = oc_subinterface.state.index + subinterface['name' ] = oc_subinterface.state.name + subinterface['enabled'] = oc_subinterface.state.enabled - subinterface_index = json_subinterface.get('state', {}).get('index') - if subinterface_index is None: continue - subinterface['index'] = int(subinterface_index) - - subinterface_name = json_subinterface.get('state', {}).get('name') - if subinterface_name is None: continue - subinterface['name'] = subinterface_name - - subinterface_enabled = json_subinterface.get('state', {}).get('enabled', False) - subinterface['enabled'] = bool(subinterface_enabled) - - VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - - # TODO: implement support for multiple IP addresses per subinterface - - IPV4_FIELDS = ('ipv4', 'openconfig-if-ip:ipv4', 'ociip:ipv4') - json_ipv4 = dict_get_first(json_subinterface, IPV4_FIELDS, default={}) - - IPV4_ADDRESSES_FIELDS = ('addresses', 'openconfig-if-ip:addresses', 'ociip:addresses') - json_ipv4_addresses = dict_get_first(json_ipv4, IPV4_ADDRESSES_FIELDS, default={}) - - IPV4_ADDRESS_FIELDS = ('address', 'openconfig-if-ip:address', 'ociip:address') - json_ipv4_address_list : List[Dict] = dict_get_first(json_ipv4_addresses, IPV4_ADDRESS_FIELDS, default=[]) - - #ipv4_addresses = [] - for json_ipv4_address in json_ipv4_address_list: - #LOGGER.info('json_ipv4_address = {:s}'.format(json.dumps(json_ipv4_address))) + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) + entries.append((entry_subinterface_key, subinterface)) - STATE_FIELDS = ('state', 'openconfig-if-ip:state', 'ociip:state') - json_ipv4_address_state = dict_get_first(json_ipv4_address, STATE_FIELDS, default={}) + #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') + #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - #ipv4_address = {} + #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') + #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - #ORIGIN_FIELDS = ('origin', 'openconfig-if-ip:origin', 'ociip:origin') - #ipv4_address_origin = dict_get_first(json_ipv4_address_state, ORIGIN_FIELDS, default={}) - #if ipv4_address_origin is not None: ipv4_address['origin'] = ipv4_address_origin + #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') + #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - IP_FIELDS = ('ip', 'openconfig-if-ip:ip', 'ociip:ip') - ipv4_address_ip = dict_get_first(json_ipv4_address_state, IP_FIELDS) - #if ipv4_address_ip is not None: ipv4_address['address_ip'] = ipv4_address_ip - if ipv4_address_ip is not None: subinterface['address_ip'] = ipv4_address_ip + #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') + #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - PREFIX_FIELDS = ('prefix-length', 'openconfig-if-ip:prefix-length', 'ociip:prefix-length') - ipv4_address_prefix = dict_get_first(json_ipv4_address_state, PREFIX_FIELDS) - #if ipv4_address_prefix is not None: ipv4_address['address_prefix'] = int(ipv4_address_prefix) - if ipv4_address_prefix is not None: subinterface['address_prefix'] = int(ipv4_address_prefix) + #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') + #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) + #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - #if len(ipv4_address) == 0: continue - #ipv4_addresses.append(ipv4_address) + for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): + #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #subinterface['ipv4_addresses'] = ipv4_addresses - - if len(subinterface) == 0: continue - resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) - response.append((resource_key, subinterface)) - - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) - - return response - - def parse_counters(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse_counters] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) - - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse_counters] json_interface = {:s}'.format(json.dumps(json_interface))) - - interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) + address_ipv4 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) + entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) + entries.append((entry_address_ipv4_key, address_ipv4)) - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) + for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): + #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #LOGGER.info('[parse_counters] interface = {:s}'.format(str(interface))) + address_ipv6 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) + entries.append((entry_address_ipv6_key, address_ipv6)) - return response + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index a45dc9e7f..1c2cfc17a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -14,67 +14,51 @@ import json, logging from typing import Any, Dict, List, Tuple +import pyangbind.lib.pybindJSON as pybindJSON +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) +#pylint: disable=abstract-method class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' - def get_path(self) -> str: return '/interfaces/interface/state/counters' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse] json_interface = {:s}'.format(json.dumps(json_interface))) + counters = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + )) interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) - - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) - - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) - - #LOGGER.info('[parse] interface = {:s}'.format(str(interface))) + interface['name'] = oc_interface.name + + interface_counters = oc_interface.state.counters + interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts + interface['in-discards' ] = interface_counters.in_discards + interface['in-errors' ] = interface_counters.in_errors + interface['in-fcs-errors' ] = interface_counters.in_fcs_errors + interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts + interface['in-octets' ] = interface_counters.in_octets + interface['in-pkts' ] = interface_counters.in_pkts + interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts + interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts + interface['out-discards' ] = interface_counters.out_discards + interface['out-errors' ] = interface_counters.out_errors + interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts + interface['out-octets' ] = interface_counters.out_octets + interface['out-pkts' ] = interface_counters.out_pkts + interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + + LOGGER.info('interface = {:s}'.format(str(interface))) if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + counters.append(('/interface[{:s}]'.format(interface['name']), interface)) - return response + return counters diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index aed821a06..c29ed263a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -13,14 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' - def get_path(self) -> str: return '/network-instances/network-instance' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: ni_name = str(resource_value['name']) # test-svc @@ -58,5 +60,10 @@ class NetworkInstanceHandler(_Handler): } def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) response = [] return response + +openconfig-network-instance:network-instance \ No newline at end of file diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 30343ac28..8cf704e29 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -23,8 +23,20 @@ def get_schema(resource_key : str): resource_key = RE_REMOVE_NAMESPACES.sub('/', resource_key) return resource_key -def dict_get_first(d : Dict, field_names : Iterable[str], default=None) -> Any: - for field_name in field_names: - if field_name not in d: continue - return d[field_name] +def container_get_first( + container : Dict[str, Any], key_name : str, namespace : Optional[str]=None, namespaces : Iterable[str]=tuple(), + default : Optional[Any] = None +) -> Any: + value = container.get(key_name) + if value is not None: return value + + if namespace is not None: + if len(namespaces) > 0: + raise Exception('At maximum, one of namespace or namespaces can be specified') + namespaces = (namespace,) + + for namespace in namespaces: + namespace_key_name = '{:s}:{:s}'.format(namespace, key_name) + if namespace_key_name in container: return container[namespace_key_name] + return default diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 39cd7c66a..6d54ef28d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -46,9 +46,10 @@ RESOURCE_KEY_MAPPER = { } PATH_MAPPER = { - '/components' : comph.get_path(), - '/interfaces' : ifaceh.get_path(), - '/network-instances' : nih.get_path(), + '/components' : comph.get_path(), + '/components/component' : comph.get_path(), + '/interfaces' : ifaceh.get_path(), + '/network-instances' : nih.get_path(), } RESOURCE_KEY_TO_HANDLER = { @@ -88,9 +89,9 @@ def get_handler( path_schema = PATH_MAPPER.get(path_schema, path_schema) handler = PATH_TO_HANDLER.get(path_schema) if handler is None and raise_if_not_found: - MSG = 'Handler not found: resource_key={:s} resource_key_schema={:s}' + MSG = 'Handler not found: path={:s} path_schema={:s}' # pylint: disable=broad-exception-raised - raise Exception(MSG.format(str(resource_key), str(resource_key_schema))) + raise Exception(MSG.format(str(path), str(path_schema))) return handler def get_path(resource_key : str) -> str: diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Path.py b/src/device/service/drivers/gnmi_openconfig/tools/Path.py index 40ab28dc6..2d6dc1e74 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Path.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Path.py @@ -19,8 +19,8 @@ from ..gnmi.gnmi_pb2 import Path, PathElem RE_PATH_SPLIT = re.compile(r'/(?=(?:[^\[\]]|\[[^\[\]]+\])*$)') RE_PATH_KEYS = re.compile(r'\[(.*?)\]') -def path_from_string(path='/'): - if not path: return Path(elem=[]) +def path_from_string(path='/'): #, origin='openconfig' + if not path: return Path(elem=[]) #, origin=origin if path[0] == '/': if path[-1] == '/': @@ -40,7 +40,7 @@ def path_from_string(path='/'): dict_keys = dict(x.split('=', 1) for x in elem_keys) path.append(PathElem(name=elem_name, key=dict_keys)) - return Path(elem=path) + return Path(elem=path) #, origin=origin def path_to_string(path : Path) -> str: path_parts = list() diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 4797930a1..9933cb858 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -13,9 +13,36 @@ # limitations under the License. import base64, json -from typing import Any +from typing import Any, Dict, List, Union from ..gnmi.gnmi_pb2 import TypedValue +REMOVE_NAMESPACES = ( + 'arista-intf-augments', + 'arista-netinst-augments', + 'openconfig-hercules-platform', +) + +def remove_fields(key : str) -> bool: + parts = key.split(':') + if len(parts) == 1: return False + namespace = parts[0].lower() + return namespace in REMOVE_NAMESPACES + +def recursive_remove_keys(container : Union[Dict, List, Any]) -> None: + if isinstance(container, dict): + remove_keys = [ + key + for key in container.keys() + if remove_fields(key) + ] + for key in remove_keys: + container.pop(key, None) + for value in container.values(): + recursive_remove_keys(value) + elif isinstance(container, list): + for value in container: + recursive_remove_keys(value) + def decode_value(value : TypedValue) -> Any: encoding = value.WhichOneof('value') if encoding == 'json_val': @@ -31,9 +58,13 @@ def decode_value(value : TypedValue) -> Any: raise NotImplementedError() #return value elif encoding == 'json_ietf_val': - value : str = value.json_ietf_val + str_value : str = value.json_ietf_val.decode('UTF-8') try: - return json.loads(value) + # Cleanup and normalize the records according to OpenConfig + str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + json_value = json.loads(str_value) + recursive_remove_keys(json_value) + return json_value except json.decoder.JSONDecodeError: # Assume is Base64-encoded b_b64_value = value.encode('UTF-8') diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py index 50c915582..684b9f4c3 100644 --- a/src/device/tests/test_gnmi.py +++ b/src/device/tests/test_gnmi.py @@ -16,9 +16,9 @@ import logging, os, sys, time from typing import Dict, Tuple os.environ['DEVICE_EMULATED_ONLY'] = 'YES' from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -#from device.service.driver_api._Driver import ( -# RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -#) +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -58,21 +58,21 @@ def main(): driver_settings = { 'protocol': 'gnmi', 'username': 'admin', - 'password': 'NokiaSrl1!', - 'use_tls' : True, + 'password': 'admin', + 'use_tls' : False, } - driver = GnmiOpenConfigDriver('172.100.100.102', 57400, **driver_settings) + driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) driver.Connect() #resources_to_get = [] #resources_to_get = [RESOURCE_ENDPOINTS] #resources_to_get = [RESOURCE_INTERFACES] - #resources_to_get = [RESOURCE_NETWORK_INSTANCES] + resources_to_get = [RESOURCE_NETWORK_INSTANCES] #resources_to_get = [RESOURCE_ROUTING_POLICIES] #resources_to_get = [RESOURCE_SERVICES] - #LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - #results_getconfig = driver.GetConfig(resources_to_get) - #LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) #resources_to_set = [ # network_instance('test-svc', 'L3VRF'), @@ -90,21 +90,21 @@ def main(): #results_setconfig = driver.SetConfig(resources_to_set) #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - resources_to_delete = [ - #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - - #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - - interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - - network_instance('20f66fb5', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + #resources_to_delete = [ + # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), + # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), + # + # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), + # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), + # + # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), + # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), + # + # #network_instance('20f66fb5', 'L3VRF'), + #] + #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + #results_deleteconfig = driver.DeleteConfig(resources_to_delete) + #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) time.sleep(1) diff --git a/test_gnmi.sh b/test_gnmi.sh new file mode 100755 index 000000000..d1fe36969 --- /dev/null +++ b/test_gnmi.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export PYTHONPATH=./src +python -m device.tests.test_gnmi -- GitLab From a345795694cb86ccbdace416441b32cffb42afe8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 19:02:09 +0000 Subject: [PATCH 002/985] DataPlane in a box: - Added management scripts - Added ContainerLab scenario descriptor - Added README.md - Added example TFS descriptors - Added links.txt --- dataplane-in-a-box/.gitignore | 2 + dataplane-in-a-box/README.md | 121 ++++++++++++++++ dataplane-in-a-box/arista.clab.yml | 54 ++++++++ dataplane-in-a-box/clab-deploy.sh | 17 +++ dataplane-in-a-box/clab-destroy.sh | 18 +++ dataplane-in-a-box/clab-inspect.sh | 17 +++ dataplane-in-a-box/clab-load-image.sh | 19 +++ dataplane-in-a-box/clab-pull-images.sh | 18 +++ dataplane-in-a-box/dc-2-dc-l3-service.json | 37 +++++ dataplane-in-a-box/deploy_specs.sh | 154 +++++++++++++++++++++ dataplane-in-a-box/links.json | 136 ++++++++++++++++++ dataplane-in-a-box/links.txt | 8 ++ dataplane-in-a-box/topology.json | 91 ++++++++++++ 13 files changed, 692 insertions(+) create mode 100644 dataplane-in-a-box/.gitignore create mode 100644 dataplane-in-a-box/README.md create mode 100644 dataplane-in-a-box/arista.clab.yml create mode 100755 dataplane-in-a-box/clab-deploy.sh create mode 100755 dataplane-in-a-box/clab-destroy.sh create mode 100755 dataplane-in-a-box/clab-inspect.sh create mode 100755 dataplane-in-a-box/clab-load-image.sh create mode 100755 dataplane-in-a-box/clab-pull-images.sh create mode 100644 dataplane-in-a-box/dc-2-dc-l3-service.json create mode 100755 dataplane-in-a-box/deploy_specs.sh create mode 100644 dataplane-in-a-box/links.json create mode 100644 dataplane-in-a-box/links.txt create mode 100644 dataplane-in-a-box/topology.json diff --git a/dataplane-in-a-box/.gitignore b/dataplane-in-a-box/.gitignore new file mode 100644 index 000000000..5de716bdd --- /dev/null +++ b/dataplane-in-a-box/.gitignore @@ -0,0 +1,2 @@ +clab-arista/ +.arista.clab.yml.bak diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md new file mode 100644 index 000000000..45e5dc5e0 --- /dev/null +++ b/dataplane-in-a-box/README.md @@ -0,0 +1,121 @@ +# DataPlane-in-a-Box - Control an Emulated DataPlane through TeraFlowSDN + +## Emulated DataPlane Deployment +- ContainerLab +- Scenario +- Descriptor + +## TeraFlowSDN Deployment +```bash +cd ~/tfs-ctrl +source dataplane-in-a-box/deploy_specs.sh +./deploy/all.sh +``` + +# ContainerLab - Arista cEOS - Commands + +## Download and install ContainerLab +```bash +sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 +``` + +## Deploy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml +``` + +## Inspect scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml +``` + +## Destroy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak +``` + +## Access cEOS Bash +```bash +docker exec -it clab-arista-wan1 bash +``` + +## Access cEOS CLI +```bash +docker exec -it clab-arista-wan1 Cli +``` + +## Configure ContainerLab clients +```bash +docker exec -it clab-arista-client1 bash + ip address add 192.168.1.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.1.1 + ip route add 192.168.3.0/24 via 192.168.1.1 + ping 192.168.2.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client2 bash + ip address add 192.168.2.10/24 dev eth1 + ip route add 192.168.1.0/24 via 192.168.2.1 + ip route add 192.168.3.0/24 via 192.168.2.1 + ping 192.168.1.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client3 bash + ip address add 192.168.3.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.3.1 + ip route add 192.168.3.0/24 via 192.168.3.1 + ping 192.168.2.10 + ping 192.168.3.10 +``` + +## Install gNMIc +```bash +sudo bash -c "$(curl -sL https://get-gnmic.kmrd.dev)" +``` + +## gNMI Capabilities request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure capabilities +``` + +## gNMI Get request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path / > wan1.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /interfaces/interface > wan1-ifaces.json +``` + +## gNMI Set request +```bash +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --update-path /system/config/hostname --update-value srl11 +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /system/config/hostname +``` + +## Subscribe request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf subscribe --path /interfaces/interface[name=Management0]/state/ + +# In another terminal, you can generate traffic opening SSH connection +ssh admin@clab-arista-wan1 +``` + +# Check configurations done: +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/network-instances' > wan1-nis.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/interfaces' > wan1-ifs.json +``` + +# Delete elements: +```bash +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/network-instances/network-instance[name=b19229e8]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/1]/subinterfaces/subinterface[index=0]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/2]/subinterfaces/subinterface[index=0]' +``` + +# Run gNMI Driver in standalone mode (advanced) +```bash +PYTHONPATH=./src python -m src.device.tests.test_gnmi +``` diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml new file mode 100644 index 000000000..9a8bff73f --- /dev/null +++ b/dataplane-in-a-box/arista.clab.yml @@ -0,0 +1,54 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TFS - Arista devices + Linux clients + +name: arista + +mgmt: + network: mgmt-net + ipv4-subnet: 172.20.20.0/24 + +topology: + kinds: + arista_ceos: + kind: arista_ceos + image: ceos:4.30.4M + linux: + kind: linux + image: ghcr.io/hellt/network-multitool:latest + + nodes: + wan1: + kind: arista_ceos + mgmt-ipv4: 172.20.20.101 + ports: [6001:6030] + wan2: + kind: arista_ceos + mgmt-ipv4: 172.20.20.102 + ports: [6002:6030] + + client1: + kind: linux + mgmt-ipv4: 172.20.20.201 + ports: [2201:22] + client2: + kind: linux + mgmt-ipv4: 172.20.20.202 + ports: [2202:22] + + links: + - endpoints: ["wan1:eth1", "wan2:eth1"] + - endpoints: ["client1:eth1", "wan1:eth10"] + - endpoints: ["client2:eth1", "wan2:eth10"] diff --git a/dataplane-in-a-box/clab-deploy.sh b/dataplane-in-a-box/clab-deploy.sh new file mode 100755 index 000000000..2b8e49a07 --- /dev/null +++ b/dataplane-in-a-box/clab-deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-destroy.sh b/dataplane-in-a-box/clab-destroy.sh new file mode 100755 index 000000000..4030239dc --- /dev/null +++ b/dataplane-in-a-box/clab-destroy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak diff --git a/dataplane-in-a-box/clab-inspect.sh b/dataplane-in-a-box/clab-inspect.sh new file mode 100755 index 000000000..02024ec47 --- /dev/null +++ b/dataplane-in-a-box/clab-inspect.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh new file mode 100755 index 000000000..87e666422 --- /dev/null +++ b/dataplane-in-a-box/clab-load-image.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download image from Arista account > software downloads + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh new file mode 100755 index 000000000..8f2805c6b --- /dev/null +++ b/dataplane-in-a-box/clab-pull-images.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker pull ghcr.io/hellt/network-multitool:latest +#docker pull ghcr.io/nokia/srlinux:23.7.2 +#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/dc-2-dc-l3-service.json new file mode 100644 index 000000000..cb9ef972e --- /dev/null +++ b/dataplane-in-a-box/dc-2-dc-l3-service.json @@ -0,0 +1,37 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + }, + "service_type": 1, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, + {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}}, + + {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}} + ]} + } + ] +} diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh new file mode 100755 index 000000000..1a978e3a9 --- /dev/null +++ b/dataplane-in-a-box/deploy_specs.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" + +# Uncomment to activate Monitoring +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json new file mode 100644 index 000000000..832a24fdd --- /dev/null +++ b/dataplane-in-a-box/links.json @@ -0,0 +1,136 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/links.txt b/dataplane-in-a-box/links.txt new file mode 100644 index 000000000..a61ad5398 --- /dev/null +++ b/dataplane-in-a-box/links.txt @@ -0,0 +1,8 @@ +https://containerlab.dev/manual/multi-node/#exposing-services +https://containerlab.dev/manual/multi-node/#bridging +https://containerlab.dev/manual/kinds/bridge/ +https://containerlab.dev/lab-examples/ext-bridge/ + +https://containerlab.dev/manual/kinds/ceos/ +https://containerlab.dev/lab-examples/srl-ceos/#__tabbed_2_2 +https://github.com/srl-labs/containerlab/blob/main/lab-examples/srlceos01/srlceos01.clab.yml diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/topology.json new file mode 100644 index 000000000..42752235d --- /dev/null +++ b/dataplane-in-a-box/topology.json @@ -0,0 +1,91 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + } + ], + "links": [] +} -- GitLab From 48c2413c0b6bd58a4a1bc571415175aed133120f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:25:34 +0000 Subject: [PATCH 003/985] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Added unitary tests and scripts - Enhanced reporting of capabilities - Migrated Component and Interface code to libyang - Migrating NetworkInstance code to libyang - Disabled unneeded log messages - Temporarily disabled telemetry - Added LibYang-based YANG handler - Added helper methods --- ...un_tests_locally-device-gnmi-openconfig.sh | 25 + .../gnmi_openconfig/GnmiSessionHandler.py | 65 +- .../gnmi_openconfig/handlers/Component.py | 39 +- .../gnmi_openconfig/handlers/Interface.py | 276 ++++---- .../handlers/NetworkInstance.py | 145 ++++- .../drivers/gnmi_openconfig/handlers/Tools.py | 21 +- .../gnmi_openconfig/handlers/YangHandler.py | 109 ++++ .../gnmi_openconfig/handlers/_Handler.py | 9 +- .../gnmi_openconfig/handlers/__init__.py | 24 +- .../gnmi_openconfig/tools/Capabilities.py | 17 +- .../drivers/gnmi_openconfig/tools/Value.py | 2 +- src/device/tests/test_gnmi.py | 115 ---- .../tests/test_unitary_gnmi_openconfig.py | 616 ++++++++++++++++++ 13 files changed, 1149 insertions(+), 314 deletions(-) create mode 100755 scripts/run_tests_locally-device-gnmi-openconfig.sh create mode 100644 src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py delete mode 100644 src/device/tests/test_gnmi.py create mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh new file mode 100755 index 000000000..d81684da1 --- /dev/null +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc + +# Run unitary tests and analyze coverage of code at same time +# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + device/tests/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 6f80ee82f..d9f73c958 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -19,12 +19,13 @@ from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk from .gnmi.gnmi_pb2_grpc import gNMIStub from .gnmi.gnmi_pb2 import Encoding, GetRequest, SetRequest, UpdateResult # pylint: disable=no-name-in-module from .handlers import ALL_RESOURCE_KEYS, compose, get_path, parse -from .tools.Capabilities import get_supported_encodings +from .handlers.YangHandler import YangHandler +from .tools.Capabilities import check_capabilities from .tools.Channel import get_grpc_channel from .tools.Path import path_from_string, path_to_string #, compose_path from .tools.Subscriptions import Subscriptions from .tools.Value import decode_value #, value_exists -from .MonitoringThread import MonitoringThread +#from .MonitoringThread import MonitoringThread class GnmiSessionHandler: def __init__(self, address : str, port : int, settings : Dict, logger : logging.Logger) -> None: @@ -39,12 +40,20 @@ class GnmiSessionHandler: self._use_tls = settings.get('use_tls', False) self._channel : Optional[grpc.Channel] = None self._stub : Optional[gNMIStub] = None - self._monit_thread = None - self._supported_encodings = None + self._yang_handler = YangHandler() + #self._monit_thread = None self._subscriptions = Subscriptions() self._in_subscriptions = queue.Queue() self._out_samples = queue.Queue() + def __del__(self) -> None: + self._logger.warning('Destroying YangValidator...') + self._logger.warning('yang_validator.data:') + for path, dnode in self._yang_handler.get_data_paths().items(): + self._logger.warning(' {:s}: {:s}'.format(str(path), json.dumps(dnode.print_dict()))) + self._yang_handler.destroy() + self._logger.warning('DONE') + @property def subscriptions(self): return self._subscriptions @@ -58,18 +67,17 @@ class GnmiSessionHandler: with self._lock: self._channel = get_grpc_channel(self._address, self._port, self._use_tls, self._logger) self._stub = gNMIStub(self._channel) - self._supported_encodings = get_supported_encodings( - self._stub, self._username, self._password, timeout=120) - self._monit_thread = MonitoringThread( - self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) - self._monit_thread.start() + check_capabilities(self._stub, self._username, self._password, timeout=120) + #self._monit_thread = MonitoringThread( + # self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) + #self._monit_thread.start() self._connected.set() def disconnect(self): if not self._connected.is_set(): return with self._lock: - self._monit_thread.stop() - self._monit_thread.join() + #self._monit_thread.stop() + #self._monit_thread.join() self._channel.close() self._connected.clear() @@ -87,9 +95,9 @@ class GnmiSessionHandler: str_resource_name = 'resource_key[#{:d}]'.format(i) try: chk_string(str_resource_name, resource_key, allow_empty=False) - self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) + #self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) str_path = get_path(resource_key) - self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) + #self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) get_request.path.append(path_from_string(str_path)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception parsing {:s}: {:s}' @@ -130,7 +138,7 @@ class GnmiSessionHandler: value = decode_value(update.val) #resource_key_tuple[1] = value #resource_key_tuple[2] = True - results.extend(parse(str_path, value)) + results.extend(parse(str_path, value, self._yang_handler)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception processing update {:s}' self._logger.exception(MSG.format(grpc_message_to_json_string(update))) @@ -159,17 +167,17 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=False) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=False) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_list = set_request.update #if exists else set_request.replace set_request_entry = set_request_list.add() set_request_entry.path.CopyFrom(path_from_string(str_path)) @@ -228,18 +236,19 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple #if not exists: continue if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=True) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + # pylint: disable=unused-variable + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=True) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_entry = set_request.delete.add() set_request_entry.CopyFrom(path_from_string(str_path)) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index cddf40d56..73728192f 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,37 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging #, json -import pyangbind.lib.pybindJSON as pybindJSON +import json, logging # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}' #pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' def get_path(self) -> str: return '/openconfig-platform:components' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) - oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') - #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) + yang_components_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_components_path, json_data, fmt='json') entries = [] - for component_key, oc_component in oc_components.component.items(): - #LOGGER.info('component_key={:s} oc_component={:s}'.format( - # component_key, pybindJSON.dumps(oc_component, mode='ietf') - #)) + for component in json_data_valid['components']['component']: + LOGGER.debug('component={:s}'.format(str(component))) - component_name = oc_component.config.name + component_name = component['name'] + #component_config = component.get('config', {}) - component_type = oc_component.state.type + #yang_components : libyang.DContainer = yang_handler.get_data_path(yang_components_path) + #yang_component_path = 'component[name="{:s}"]'.format(component_name) + #yang_component : libyang.DContainer = yang_components.create_path(yang_component_path) + #yang_component.merge_data_dict(component, strict=True, validate=False) + + component_state = component.get('state', {}) + component_type = component_state.get('type') + if component_type is None: continue component_type = component_type.split(':')[-1] if component_type not in {'PORT'}: continue @@ -58,8 +65,6 @@ class ComponentHandler(_Handler): KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED: PATH_IF_CTR.format(interface_name, 'out-pkts' ), } - if len(endpoint) == 0: continue - entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 77310d51d..f28cdcf36 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -24,9 +24,11 @@ class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' def get_path(self) -> str: return '/openconfig-interfaces:interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - if_name = str (resource_value['name' ]) # ethernet-1/1 - sif_index = int (resource_value.get('sub_if_index' , 0 )) # 0 + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + if_name = get_str(resource_value, 'name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sub_if_index', 0) # 0 if delete: PATH_TMPL = '/interfaces/interface[name={:s}]/subinterfaces/subinterface[index={:d}]' @@ -34,118 +36,166 @@ class InterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - if_enabled = bool(resource_value.get('enabled' , True)) # True/False - sif_enabled = bool(resource_value.get('sub_if_enabled' , True)) # True/False - sif_ipv4_enabled = bool(resource_value.get('sub_if_ipv4_enabled', True)) # True/False - sif_ipv4_address = str (resource_value['sub_if_ipv4_address' ]) # 172.16.0.1 - sif_ipv4_prefix = int (resource_value['sub_if_ipv4_prefix' ]) # 24 + if_enabled = get_bool(resource_value, 'enabled', True) # True/False + sif_enabled = get_bool(resource_value, 'sub_if_enabled', True) # True/False + sif_vlan_id = get_int (resource_value, 'sif_vlan_id', ) # 127 + sif_ipv4_enabled = get_bool(resource_value, 'sub_if_ipv4_enabled', True) # True/False + sif_ipv4_address = get_str (resource_value, 'sub_if_ipv4_address' ) # 172.16.0.1 + sif_ipv4_prefix = get_int (resource_value, 'sub_if_ipv4_prefix' ) # 24 + + yang_ifs : libyang.DContainer = yang_handler.get_data_path('/openconfig-interfaces:interfaces') + yang_if_path = 'interface[name="{:s}"]'.format(if_name) + yang_if : libyang.DContainer = yang_ifs.create_path(yang_if_path) + yang_if.create_path('config/name', if_name ) + if if_enabled is not None: yang_if.create_path('config/enabled', if_enabled) + + yang_sifs : libyang.DContainer = yang_if.create_path('subinterfaces') + yang_sif_path = 'subinterface[index="{:d}"]'.format(sif_index) + yang_sif : libyang.DContainer = yang_sifs.create_path(yang_sif_path) + yang_sif.create_path('config/index', sif_index) + if sif_enabled is not None: yang_sif.create_path('config/enabled', sif_enabled) + + if sif_vlan_id is not None: + yang_subif_vlan : libyang.DContainer = yang_sif.create_path('openconfig-vlan:vlan') + yang_subif_vlan.create_path('match/single-tagged/config/vlan-id', sif_vlan_id) + + yang_ipv4 : libyang.DContainer = yang_sif.create_path('openconfig-if-ip:ipv4') + if sif_ipv4_enabled is not None: yang_ipv4.create_path('config/enabled', sif_ipv4_enabled) + + if sif_ipv4_address is not None: + yang_ipv4_addrs : libyang.DContainer = yang_ipv4.create_path('addresses') + yang_ipv4_addr_path = 'address[ip="{:s}"]'.format(sif_ipv4_address) + yang_ipv4_addr : libyang.DContainer = yang_ipv4_addrs.create_path(yang_ipv4_addr_path) + yang_ipv4_addr.create_path('config/ip', sif_ipv4_address) + yang_ipv4_addr.create_path('config/prefix-length', sif_ipv4_prefix ) str_path = '/interfaces/interface[name={:s}]'.format(if_name) - str_data = json.dumps({ - 'name': if_name, - 'config': {'name': if_name, 'enabled': if_enabled}, - 'subinterfaces': { - 'subinterface': { - 'index': sif_index, - 'config': {'index': sif_index, 'enabled': sif_enabled}, - 'ipv4': { - 'config': {'enabled': sif_ipv4_enabled}, - 'addresses': { - 'address': { - 'ip': sif_ipv4_address, - 'config': {'ip': sif_ipv4_address, 'prefix_length': sif_ipv4_prefix}, - } - } - } - } - } - }) + str_data = yang_if.print_mem('json') + json_data = json.loads(str_data) + json_data = json_data['openconfig-interfaces:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') entries = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - #)) - - interface = {} - interface['name'] = oc_interface.config.name - - interface_type = oc_interface.config.type - interface_type = interface_type.replace('ianaift:', '') - interface_type = interface_type.replace('iana-if-type:', '') - interface['type'] = interface_type - - interface['mtu' ] = oc_interface.config.mtu - interface['enabled' ] = oc_interface.config.enabled - interface['description' ] = oc_interface.config.description - interface['admin-status'] = oc_interface.state.admin_status - interface['oper-status' ] = oc_interface.state.oper_status - interface['management' ] = oc_interface.state.management - - entry_interface_key = '/interface[{:s}]'.format(interface['name']) - entries.append((entry_interface_key, interface)) - - for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): - #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( - # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') - #)) - - subinterface = {} - subinterface['index' ] = oc_subinterface.state.index - subinterface['name' ] = oc_subinterface.state.name - subinterface['enabled'] = oc_subinterface.state.enabled - - entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) - entries.append((entry_subinterface_key, subinterface)) - - #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): - #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv4 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) - entries.append((entry_address_ipv4_key, address_ipv4)) - - for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): - #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv6 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) - entries.append((entry_address_ipv6_key, address_ipv6)) + for interface in json_data_valid['interfaces']['interface']: + LOGGER.debug('interface={:s}'.format(str(interface))) + + interface_name = interface['name'] + interface_config = interface.get('config', {}) + + #yang_interfaces : libyang.DContainer = yang_handler.get_data_path(yang_interfaces_path) + #yang_interface_path = 'interface[name="{:s}"]'.format(interface_name) + #yang_interface : libyang.DContainer = yang_interfaces.create_path(yang_interface_path) + #yang_interface.merge_data_dict(interface, strict=True, validate=False) + + interface_state = interface.get('state', {}) + interface_type = interface_state.get('type') + if interface_type is None: continue + interface_type = interface_type.split(':')[-1] + if interface_type not in {'ethernetCsmacd'}: continue + + _interface = { + 'name' : interface_name, + 'type' : interface_type, + 'mtu' : interface_state['mtu'], + 'ifindex' : interface_state['ifindex'], + 'admin-status' : interface_state['admin-status'], + 'oper-status' : interface_state['oper-status'], + 'management' : interface_state['management'], + } + if 'description' in interface_config: + _interface['description'] = interface_config['description'] + if 'enabled' in interface_config: + _interface['enabled'] = interface_config['enabled'] + if 'hardware-port' in interface_state: + _interface['hardware-port'] = interface_state['hardware-port'] + if 'transceiver' in interface_state: + _interface['transceiver'] = interface_state['transceiver'] + + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) + + if interface_type == 'ethernetCsmacd': + ethernet_state = interface['ethernet']['state'] + + _ethernet = { + 'mac-address' : ethernet_state['mac-address'], + 'hw-mac-address' : ethernet_state['hw-mac-address'], + 'port-speed' : ethernet_state['port-speed'].split(':')[-1], + 'negotiated-port-speed' : ethernet_state['negotiated-port-speed'].split(':')[-1], + } + entry_ethernet_key = '{:s}/ethernet'.format(entry_interface_key) + entries.append((entry_ethernet_key, _ethernet)) + + subinterfaces = interface.get('subinterfaces', {}).get('subinterface', []) + for subinterface in subinterfaces: + LOGGER.debug('subinterface={:s}'.format(str(subinterface))) + + subinterface_index = subinterface['index'] + subinterface_state = subinterface.get('state', {}) + + _subinterface = {'index': subinterface_index} + if 'name' in subinterface_state: + _subinterface['name'] = subinterface_state['name'] + if 'enabled' in subinterface_state: + _subinterface['enabled'] = subinterface_state['enabled'] + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface_index) + entries.append((entry_subinterface_key, _subinterface)) + + if 'vlan' in subinterface: + vlan = subinterface['vlan'] + vlan_match = vlan['match'] + + single_tagged = vlan_match.pop('single-tagged', None) + if single_tagged is not None: + single_tagged_config = single_tagged['config'] + vlan_id = single_tagged_config['vlan-id'] + + _vlan = {'vlan_id': vlan_id} + entry_vlan_key = '{:s}/vlan[single:{:s}]'.format(entry_subinterface_key, vlan_id) + entries.append((entry_vlan_key, _vlan)) + + if len(vlan_match) > 0: + raise Exception('Unsupported VLAN schema: {:s}'.format(str(vlan))) + + ipv4_addresses = subinterface.get('ipv4', {}).get('addresses', {}).get('address', []) + for ipv4_address in ipv4_addresses: + LOGGER.debug('ipv4_address={:s}'.format(str(ipv4_address))) + + ipv4_address_ip = ipv4_address['ip'] + ipv4_address_state = ipv4_address.get('state', {}) + + _ipv4_address = {'ip': ipv4_address_ip} + if 'origin' in ipv4_address_state: + _ipv4_address['origin'] = ipv4_address_state['origin'] + if 'prefix-length' in ipv4_address_state: + _ipv4_address['prefix'] = ipv4_address_state['prefix-length'] + + entry_ipv4_address_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, ipv4_address_ip) + entries.append((entry_ipv4_address_key, _ipv4_address)) + + ipv6_addresses = subinterface.get('ipv6', {}).get('addresses', {}).get('address', []) + for ipv6_address in ipv6_addresses: + LOGGER.debug('ipv6_address={:s}'.format(str(ipv6_address))) + + ipv6_address_ip = ipv6_address['ip'] + ipv6_address_state = ipv6_address.get('state', {}) + + _ipv6_address = {'ip': ipv6_address_ip} + if 'origin' in ipv6_address_state: + _ipv6_address['origin'] = ipv6_address_state['origin'] + if 'prefix-length' in ipv6_address_state: + _ipv6_address['prefix'] = ipv6_address_state['prefix-length'] + + entry_ipv6_address_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, ipv6_address_ip) + entries.append((entry_ipv6_address_key, _ipv6_address)) return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index c29ed263a..0b4d15745 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,20 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging +import operator from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +MAP_NETWORK_INSTANCE_TYPE = { + # special routing instance; acts as default/global routing instance for a network device + 'DEFAULT': 'openconfig-network-instance-types:DEFAULT_INSTANCE', + + # private L3-only routing instance; formed of one or more RIBs + 'L3VRF': 'openconfig-network-instance-types:L3VRF', + + # private L2-only switch instance; formed of one or more L2 forwarding tables + 'L2VSI': 'openconfig-network-instance-types:L2VSI', + + # private L2-only forwarding instance; point to point connection between two endpoints + 'L2P2P': 'openconfig-network-instance-types:L2P2P', + + # private Layer 2 and Layer 3 forwarding instance + 'L2L3': 'openconfig-network-instance-types:L2L3', +} + class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' def get_path(self) -> str: return '/openconfig-network-instance:network-instances' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name']) # test-svc + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -33,15 +53,11 @@ class NetworkInstanceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - ni_type = str(resource_value['type']) # L3VRF / L2VSI / ... + ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... + ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # not works: [FailedPrecondition] unsupported identifier 'DIRECTLY_CONNECTED' - #protocols = [self._compose_directly_connected()] + # 'DIRECTLY_CONNECTED' is implicitly added - MAP_OC_NI_TYPE = { - 'L3VRF': 'openconfig-network-instance-types:L3VRF', - } - ni_type = MAP_OC_NI_TYPE.get(ni_type, ni_type) str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) str_data = json.dumps({ @@ -51,19 +67,92 @@ class NetworkInstanceHandler(_Handler): }) return str_path, str_data - def _compose_directly_connected(self, name=None, enabled=True) -> Dict: - identifier = 'DIRECTLY_CONNECTED' - if name is None: name = 'DIRECTLY_CONNECTED' - return { - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': enabled}, - } - - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - return response - -openconfig-network-instance:network-instance \ No newline at end of file + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + # Arista Parsing Fixes: + # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 + # overwrite with .../hellos/config/hello-interval + network_instances = json_data.get('openconfig-network-instance:network-instance', []) + for network_instance in network_instances: + if network_instance['name'] != 'default': continue + mpls_rsvp_te = network_instance.get('mpls', {}).get('signaling-protocols', {}).get('rsvp-te', {}) + mpls_rsvp_te_hellos = mpls_rsvp_te.get('global', {}).get('hellos', {}) + hello_interval = mpls_rsvp_te_hellos.get('config', {}).get('hello-interval', 9000) + mpls_rsvp_te_hellos.get('state', {})['hello-interval'] = hello_interval + + yang_network_instances_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_network_instances_path, json_data, fmt='json', strict=False) + + entries = [] + for network_instance in json_data_valid['network-instances']['network-instance']: + LOGGER.debug('network_instance={:s}'.format(str(network_instance))) + ni_name = network_instance['name'] + + ni_config = network_instance['config'] + ni_type = ni_config['type'].split(':')[-1] + + _net_inst = {'name': ni_name, 'type': ni_type} + entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) + entries.append((entry_net_inst_key, _net_inst)) + + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) + for ni_protocol in ni_protocols: + ni_protocol_id = ni_protocol['identifier'].split(':')[-1] + ni_protocol_name = ni_protocol['name'] + + _protocol = {'id': ni_protocol_id, 'name': ni_protocol_name} + entry_protocol_key = '{:s}/protocol[{:s}]'.format(entry_net_inst_key, ni_protocol_id) + entries.append((entry_protocol_key, _protocol)) + + if ni_protocol_id == 'STATIC': + static_routes = ni_protocol.get('static-routes', {}).get('static', []) + for static_route in static_routes: + static_route_prefix = static_route['prefix'] + + next_hops = static_route.get('next-hops', {}).get('next-hop', []) + _next_hops = [ + { + 'index' : next_hop['index'], + 'gateway': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], + } + for next_hop in next_hops + ] + _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + + _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + entry_static_route_key = '{:s}/static_routes[{:s}]'.format( + entry_protocol_key, static_route_prefix + ) + entries.append((entry_static_route_key, _static_route)) + + ni_tables = network_instance.get('tables', {}).get('table', []) + for ni_table in ni_tables: + ni_table_protocol = ni_table['protocol'].split(':')[-1] + ni_table_address_family = ni_table['address-family'].split(':')[-1] + _table = {'protocol': ni_table_protocol, 'address_family': ni_table_address_family} + entry_table_key = '{:s}/table[{:s},{:s}]'.format( + entry_net_inst_key, ni_table_protocol, ni_table_address_family + ) + entries.append((entry_table_key, _table)) + + ni_vlans = network_instance.get('vlans', {}).get('vlan', []) + for ni_vlan in ni_vlans: + ni_vlan_id = ni_vlan['vlan-id'] + + #ni_vlan_config = ni_vlan['config'] + ni_vlan_state = ni_vlan['state'] + ni_vlan_name = ni_vlan_state['name'] + + _members = [ + member['state']['interface'] + for member in ni_vlan.get('members', {}).get('member', []) + ] + _vlan = {'vlan_id': ni_vlan_id, 'name': ni_vlan_name, 'members': _members} + entry_vlan_key = '{:s}/vlan[{:d}]'.format(entry_net_inst_key, ni_vlan_id) + entries.append((entry_vlan_key, _vlan)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 8cf704e29..dfb8eabaf 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable, Optional +from typing import Any, Callable, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -40,3 +40,22 @@ def container_get_first( if namespace_key_name in container: return container[namespace_key_name] return default + +def get_value( + resource_value : Dict, field_name : str, cast_func : Callable = lambda x:x, default : Optional[Any] = None +) -> Optional[Any]: + field_value = resource_value.get(field_name, default) + if field_value is not None: field_value = cast_func(field_value) + return field_value + +def get_bool(resource_value : Dict, field_name : bool, default : Optional[Any] = None) -> bool: + return get_value(resource_value, field_name, cast_func=bool, default=default) + +def get_float(resource_value : Dict, field_name : float, default : Optional[Any] = None) -> float: + return get_value(resource_value, field_name, cast_func=float, default=default) + +def get_int(resource_value : Dict, field_name : int, default : Optional[Any] = None) -> int: + return get_value(resource_value, field_name, cast_func=int, default=default) + +def get_str(resource_value : Dict, field_name : str, default : Optional[Any] = None) -> str: + return get_value(resource_value, field_name, cast_func=str, default=default) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py new file mode 100644 index 000000000..fe8672187 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py @@ -0,0 +1,109 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, libyang, logging, os +from typing import Dict, Optional + +YANG_BASE_PATH = os.path.join(os.path.dirname(__file__), '..', 'git', 'openconfig', 'public') +YANG_SEARCH_PATHS = ':'.join([ + os.path.join(YANG_BASE_PATH, 'release'), + os.path.join(YANG_BASE_PATH, 'third_party'), +]) + +YANG_MODULES = [ + 'iana-if-type', + 'openconfig-vlan-types', + + 'openconfig-interfaces', + 'openconfig-if-8021x', + 'openconfig-if-aggregate', + 'openconfig-if-ethernet-ext', + 'openconfig-if-ethernet', + 'openconfig-if-ip-ext', + 'openconfig-if-ip', + 'openconfig-if-poe', + 'openconfig-if-sdn-ext', + 'openconfig-if-tunnel', + + 'openconfig-vlan', + + 'openconfig-types', + 'openconfig-policy-types', + 'openconfig-mpls-types', + 'openconfig-network-instance-types', + 'openconfig-network-instance', + + 'openconfig-platform', + 'openconfig-platform-controller-card', + 'openconfig-platform-cpu', + 'openconfig-platform-ext', + 'openconfig-platform-fabric', + 'openconfig-platform-fan', + 'openconfig-platform-integrated-circuit', + 'openconfig-platform-linecard', + 'openconfig-platform-pipeline-counters', + 'openconfig-platform-port', + 'openconfig-platform-psu', + 'openconfig-platform-software', + 'openconfig-platform-transceiver', + 'openconfig-platform-types', +] + +LOGGER = logging.getLogger(__name__) + +class YangHandler: + def __init__(self) -> None: + self._yang_context = libyang.Context(YANG_SEARCH_PATHS) + self._loaded_modules = set() + for yang_module_name in YANG_MODULES: + LOGGER.info('Loading module: {:s}'.format(str(yang_module_name))) + self._yang_context.load_module(yang_module_name).feature_enable_all() + self._loaded_modules.add(yang_module_name) + self._data_path_instances = dict() + + def get_data_paths(self) -> Dict[str, libyang.DNode]: + return self._data_path_instances + + def get_data_path(self, path : str) -> libyang.DNode: + data_path_instance = self._data_path_instances.get(path) + if data_path_instance is None: + data_path_instance = self._yang_context.create_data_path(path) + self._data_path_instances[path] = data_path_instance + return data_path_instance + + def parse_to_dict( + self, request_path : str, json_data : Dict, fmt : str = 'json', strict : bool = True + ) -> Dict: + if fmt != 'json': raise Exception('Unsupported format: {:s}'.format(str(fmt))) + LOGGER.debug('request_path = {:s}'.format(str(request_path))) + LOGGER.debug('json_data = {:s}'.format(str(json_data))) + LOGGER.debug('format = {:s}'.format(str(fmt))) + + parent_path_parts = list(filter(lambda s: len(s) > 0, request_path.split('/'))) + for parent_path_part in reversed(parent_path_parts): + json_data = {parent_path_part: json_data} + str_data = json.dumps(json_data) + + dnode : Optional[libyang.DNode] = self._yang_context.parse_data_mem( + str_data, fmt, strict=strict, parse_only=True, #validate_present=True, #validate=True, + ) + if dnode is None: raise Exception('Unable to parse Data({:s})'.format(str(json_data))) + + parsed = dnode.print_dict() + LOGGER.debug('parsed = {:s}'.format(json.dumps(parsed))) + dnode.free() + return parsed + + def destroy(self) -> None: + self._yang_context.destroy() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py index d20c77b11..a03692d95 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Any, Dict, List, Tuple +from .YangHandler import YangHandler class _Handler: def get_resource_key(self) -> str: @@ -23,10 +24,14 @@ class _Handler: # Retrieve the OpenConfig path schema used to interrogate the device raise NotImplementedError() - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: # Compose a Set/Delete message based on the resource_key/resource_value fields, and the delete flag raise NotImplementedError() - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: # Parse a Reply from the device and return a list of resource_key/resource_value pairs raise NotImplementedError() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 6d54ef28d..38bc4db40 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES from ._Handler import _Handler from .Component import ComponentHandler @@ -23,6 +23,7 @@ from .NetworkInstance import NetworkInstanceHandler from .NetworkInstanceInterface import NetworkInstanceInterfaceHandler from .NetworkInstanceStaticRoute import NetworkInstanceStaticRouteHandler from .Tools import get_schema +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -71,7 +72,8 @@ PATH_TO_HANDLER = { } def get_handler( - resource_key : Optional[str] = None, path : Optional[str] = None, raise_if_not_found=True + resource_key : Optional[str] = None, path : Optional[str] = None, + raise_if_not_found=True ) -> Optional[_Handler]: if (resource_key is None) == (path is None): MSG = 'Exactly one of resource_key({:s}) or path({:s}) must be specified' @@ -95,10 +97,18 @@ def get_handler( return handler def get_path(resource_key : str) -> str: - return get_handler(resource_key=resource_key).get_path() + handler = get_handler(resource_key=resource_key) + return handler.get_path() -def parse(str_path : str, value : Union[Dict, List]): - return get_handler(path=str_path).parse(value) +def parse( + str_path : str, value : Union[Dict, List], yang_handler : YangHandler +) -> List[Tuple[str, Dict[str, Any]]]: + handler = get_handler(path=str_path) + return handler.parse(value, yang_handler) -def compose(resource_key : str, resource_value : Union[Dict, List], delete : bool = False) -> Tuple[str, str]: - return get_handler(resource_key=resource_key).compose(resource_key, resource_value, delete=delete) +def compose( + resource_key : str, resource_value : Union[Dict, List], + yang_handler : YangHandler, delete : bool = False +) -> Tuple[str, str]: + handler = get_handler(resource_key=resource_key) + return handler.compose(resource_key, resource_value, yang_handler, delete=delete) diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py index b90bf3db8..4c202da2c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py @@ -17,7 +17,7 @@ from common.tools.grpc.Tools import grpc_message_to_json from ..gnmi.gnmi_pb2 import CapabilityRequest # pylint: disable=no-name-in-module from ..gnmi.gnmi_pb2_grpc import gNMIStub -def get_supported_encodings( +def check_capabilities( stub : gNMIStub, username : str, password : str, timeout : Optional[int] = None ) -> Set[Union[str, int]]: metadata = [('username', username), ('password', password)] @@ -25,6 +25,17 @@ def get_supported_encodings( reply = stub.Capabilities(req, metadata=metadata, timeout=timeout) data = grpc_message_to_json(reply) + + gnmi_version = data.get('gNMI_version') + if gnmi_version is None or gnmi_version != '0.7.0': + raise Exception('Unsupported gNMI version: {:s}'.format(str(gnmi_version))) + + #supported_models = { + # supported_model['name']: supported_model['version'] + # for supported_model in data.get('supported_models', []) + #} + # TODO: check supported models and versions + supported_encodings = { supported_encoding for supported_encoding in data.get('supported_encodings', []) @@ -33,4 +44,6 @@ def get_supported_encodings( if len(supported_encodings) == 0: # pylint: disable=broad-exception-raised raise Exception('No supported encodings found') - return supported_encodings + if 'JSON_IETF' not in supported_encodings: + # pylint: disable=broad-exception-raised + raise Exception('JSON_IETF encoding not supported') diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 9933cb858..73e43b87c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -61,7 +61,7 @@ def decode_value(value : TypedValue) -> Any: str_value : str = value.json_ietf_val.decode('UTF-8') try: # Cleanup and normalize the records according to OpenConfig - str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + #str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') json_value = json.loads(str_value) recursive_remove_keys(json_value) return json_value diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py deleted file mode 100644 index 684b9f4c3..000000000 --- a/src/device/tests/test_gnmi.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, os, sys, time -from typing import Dict, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | # | Name | Container ID | Image | Kind | State | IPv4 Address | IPv6 Address | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | 1 | clab-tfs-scenario-client1 | a8d48ec3265a | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.201/24 | N/A | -# | 2 | clab-tfs-scenario-client2 | fc88436d2b32 | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.202/24 | N/A | -# | 3 | clab-tfs-scenario-srl1 | b995b9bdadda | ghcr.io/nokia/srlinux | srl | running | 172.100.100.101/24 | N/A | -# | 4 | clab-tfs-scenario-srl2 | aacfc38cc376 | ghcr.io/nokia/srlinux | srl | running | 172.100.100.102/24 | N/A | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = {'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix} - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def main(): - driver_settings = { - 'protocol': 'gnmi', - 'username': 'admin', - 'password': 'admin', - 'use_tls' : False, - } - driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) - driver.Connect() - - #resources_to_get = [] - #resources_to_get = [RESOURCE_ENDPOINTS] - #resources_to_get = [RESOURCE_INTERFACES] - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - #resources_to_get = [RESOURCE_ROUTING_POLICIES] - #resources_to_get = [RESOURCE_SERVICES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #resources_to_set = [ - # network_instance('test-svc', 'L3VRF'), - # - # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/1', 0), - # - # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/2', 0), - # - # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), - # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), - #] - #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - #results_setconfig = driver.SetConfig(resources_to_set) - #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - #resources_to_delete = [ - # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - # - # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - # - # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - # - # #network_instance('20f66fb5', 'L3VRF'), - #] - #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - #results_deleteconfig = driver.DeleteConfig(resources_to_delete) - #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - time.sleep(1) - - driver.Disconnect() - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..4c2dca5d5 --- /dev/null +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -0,0 +1,616 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict, List, Tuple +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### STORAGE POPULATORS ############################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = resource_value.get('members') + continue + + +##### EXPECTED CONFIG COMPOSERS ######################################################################################## + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config + + +##### REQUEST COMPOSERS ################################################################################################ + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = {'name': ni_name, 'type': ni_type} + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} + return str_path, str_data + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[test-l3-svc]', { + 'name': 'test-l3-svc', 'type': 'L3VRF' + }), + ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +#def test_unitary_gnmi_openconfig( +# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +#) -> None: +# #resources_to_get = [] +# resources_to_get = [RESOURCE_ENDPOINTS] +# #resources_to_get = [RESOURCE_INTERFACES] +# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] +# #resources_to_get = [RESOURCE_ROUTING_POLICIES] +# #resources_to_get = [RESOURCE_SERVICES] +# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) +# results_getconfig = driver.GetConfig(resources_to_get) +# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) +# +# #resources_to_set = [ +# # network_instance('test-svc', 'L3VRF'), +# # +# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/1', 0), +# # +# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/2', 0), +# # +# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), +# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), +# #] +# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) +# #results_setconfig = driver.SetConfig(resources_to_set) +# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) +# +# #resources_to_delete = [ +# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), +# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), +# # +# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), +# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), +# # +# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), +# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), +# # +# # #network_instance('20f66fb5', 'L3VRF'), +# #] +# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) +# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) +# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From f49ad8703b3e2cd54614f4ac95d22689e9ea67ba Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:40:02 +0000 Subject: [PATCH 004/985] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Corrected basic unitary test for network instances --- .../tests/test_unitary_gnmi_openconfig.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py index 4c2dca5d5..7d33d1a71 100644 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -141,7 +141,7 @@ def populate_network_instances_storage( prefix = match.group(3) ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) continue match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) @@ -161,7 +161,7 @@ def populate_network_instances_storage( ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) ni_v_storage['vlan_id'] = vlan_id ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = resource_value.get('members') + ni_v_storage['members'] = sorted(resource_value.get('members')) continue @@ -389,6 +389,12 @@ def test_get_network_instances( populate_network_instances_storage(storage, results_getconfig) expected_getconfig = get_expected_network_instance_config(storage) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) @@ -481,6 +487,9 @@ def test_set_network_instances( 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' }) ]) + for resource_key, resource_value in expected_getconfig: + if resource_key == '/network_instance[default]/vlan[1]': + resource_value['members'] = list() LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) permitted_retries = 5 @@ -490,6 +499,12 @@ def test_set_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs == 0: break @@ -562,6 +577,12 @@ def test_del_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + expected_getconfig = get_expected_network_instance_config(storage) diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) -- GitLab From a3df08b8600e8aa1c54e27516b5784a126f881c2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:57:04 +0000 Subject: [PATCH 005/985] DataPlane-in-a-box: - Added scripts to connect to cEOS CLI - Updated cEOS version in ContainerLab descriptors --- dataplane-in-a-box/arista.clab.yml | 3 ++- dataplane-in-a-box/ceos-cli-wan1.sh | 3 +++ dataplane-in-a-box/ceos-cli-wan2.sh | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100755 dataplane-in-a-box/ceos-cli-wan1.sh create mode 100755 dataplane-in-a-box/ceos-cli-wan2.sh diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 9a8bff73f..4f3b77129 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -24,7 +24,8 @@ topology: kinds: arista_ceos: kind: arista_ceos - image: ceos:4.30.4M + #image: ceos:4.30.4M + image: ceos:4.31.1F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/ceos-cli-wan1.sh new file mode 100755 index 000000000..4ae21bcb5 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan1 Cli diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/ceos-cli-wan2.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli -- GitLab From 32d82663c45cb5a53f2980e066a838cc571b6b70 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:58:30 +0000 Subject: [PATCH 006/985] Device component - GNMI OpenConfig: - advanced development of driver - updated unitary tests and related scripts --- ...un_tests_locally-device-gnmi-openconfig.sh | 2 +- .../handlers/InterfaceCounter.py | 66 +- .../handlers/NetworkInstance.py | 34 +- .../handlers/NetworkInstanceInterface.py | 52 +- .../handlers/NetworkInstanceStaticRoute.py | 79 ++- src/device/tests/gnmi_openconfig/__init__.py | 14 + .../gnmi_openconfig/request_composers.py | 44 ++ src/device/tests/gnmi_openconfig/storage.py | 285 ++++++++ .../test_unitary_gnmi_openconfig.py | 556 +++++++++++++++ .../tests/test_unitary_gnmi_openconfig.py | 637 ------------------ 10 files changed, 1051 insertions(+), 718 deletions(-) create mode 100644 src/device/tests/gnmi_openconfig/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/request_composers.py create mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py delete mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh index d81684da1..7183b4104 100755 --- a/scripts/run_tests_locally-device-gnmi-openconfig.sh +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -22,4 +22,4 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc # Run unitary tests and analyze coverage of code at same time # helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ - device/tests/test_unitary_gnmi_openconfig.py + device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index 1c2cfc17a..d4701826c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple -import pyangbind.lib.pybindJSON as pybindJSON -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -25,40 +24,41 @@ class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - counters = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - )) + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') - interface = {} - interface['name'] = oc_interface.name - - interface_counters = oc_interface.state.counters - interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts - interface['in-discards' ] = interface_counters.in_discards - interface['in-errors' ] = interface_counters.in_errors - interface['in-fcs-errors' ] = interface_counters.in_fcs_errors - interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts - interface['in-octets' ] = interface_counters.in_octets - interface['in-pkts' ] = interface_counters.in_pkts - interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts - interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts - interface['out-discards' ] = interface_counters.out_discards - interface['out-errors' ] = interface_counters.out_errors - interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts - interface['out-octets' ] = interface_counters.out_octets - interface['out-pkts' ] = interface_counters.out_pkts - interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + entries = [] + for interface in json_data_valid['interfaces']['interface']: + LOGGER.info('interface={:s}'.format(str(interface))) + interface_name = interface['name'] + interface_counters = interface.get('state', {}).get('counters', {}) + _interface = { + 'name' : interface_name, + 'in-broadcast-pkts' : interface_counters['in_broadcast_pkts' ], + 'in-discards' : interface_counters['in_discards' ], + 'in-errors' : interface_counters['in_errors' ], + 'in-fcs-errors' : interface_counters['in_fcs_errors' ], + 'in-multicast-pkts' : interface_counters['in_multicast_pkts' ], + 'in-octets' : interface_counters['in_octets' ], + 'in-pkts' : interface_counters['in_pkts' ], + 'in-unicast-pkts' : interface_counters['in_unicast_pkts' ], + 'out-broadcast-pkts': interface_counters['out_broadcast_pkts'], + 'out-discards' : interface_counters['out_discards' ], + 'out-errors' : interface_counters['out_errors' ], + 'out-multicast-pkts': interface_counters['out_multicast_pkts'], + 'out-octets' : interface_counters['out_octets' ], + 'out-pkts' : interface_counters['out_pkts' ], + 'out-unicast-pkts' : interface_counters['out_unicast_pkts' ], + } LOGGER.info('interface = {:s}'.format(str(interface))) - if len(interface) == 0: continue - counters.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) - return counters + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 0b4d15745..1efed024c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, libyang, logging -import operator +import json, libyang, logging, operator from typing import Any, Dict, List, Tuple from ._Handler import _Handler -from .Tools import get_bool, get_int, get_str +from .Tools import get_str from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -45,7 +44,7 @@ class NetworkInstanceHandler(_Handler): def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - ni_name = get_str(resource_value, 'name') # test-svc + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -56,21 +55,32 @@ class NetworkInstanceHandler(_Handler): ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # 'DIRECTLY_CONNECTED' is implicitly added + str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) + #str_data = json.dumps({ + # 'name': ni_name, + # 'config': {'name': ni_name, 'type': ni_type}, + #}) + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni_path = 'network-instance[name="{:s}"]'.format(ni_name) + yang_ni : libyang.DContainer = yang_nis.create_path(yang_ni_path) + yang_ni.create_path('config/name', ni_name) + yang_ni.create_path('config/type', ni_type) - str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) - str_data = json.dumps({ - 'name': ni_name, - 'config': {'name': ni_name, 'type': ni_type}, - #'protocols': {'protocol': protocols}, - }) + # 'DIRECTLY_CONNECTED' is implicitly added + #'protocols': {'protocol': protocols}, + + str_data = yang_ni.print_mem('json') + LOGGER.warning('str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:network-instance'][0] + str_data = json.dumps(json_data) return str_path, str_data def parse( self, json_data : Dict, yang_handler : YangHandler ) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) # Arista Parsing Fixes: # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index 205373fca..ab105c2b0 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -12,21 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +IS_CEOS = True + class NetworkInstanceInterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/interface' - def get_path(self) -> str: return '/network-instances/network-instance/interfaces' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - if_name = str(resource_value['if_name' ]) # ethernet-1/1 - sif_index = int(resource_value['sif_index']) # 0 - if_id = '{:s}.{:d}'.format(if_name, sif_index) + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sif_index', 0) # 0 + + if IS_CEOS: + if_id = if_name + else: + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -35,12 +45,30 @@ class NetworkInstanceInterfaceHandler(_Handler): return str_path, str_data str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) - str_data = json.dumps({ - 'id': if_id, - 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, - }) + #str_data = json.dumps({ + # 'id': if_id, + # 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_ifs : libyang.DContainer = yang_ni.create_path('interfaces') + yang_ni_if_path = 'interface[id="{:s}"]'.format(if_id) + yang_ni_if : libyang.DContainer = yang_ni_ifs.create_path(yang_ni_if_path) + yang_ni_if.create_path('config/id', if_id) + yang_ni_if.create_path('config/interface', if_name) + yang_ni_if.create_path('config/subinterface', sif_index) + + str_data = yang_ni_if.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 9d75e9ac6..0343e3cba 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -12,21 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) class NetworkInstanceStaticRouteHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/static_route' - def get_path(self) -> str: return '/network-instances/network-instance/static_route' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/static_route' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - prefix = str(resource_value['prefix' ]) # '172.0.1.0/24' + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + prefix = get_str(resource_value, 'prefix') # '172.0.1.0/24' - identifier = 'STATIC' + identifier = 'openconfig-policy-types:STATIC' name = 'static' if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols' @@ -35,27 +39,56 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = str(resource_value['next_hop' ]) # '172.0.0.1' - next_hop_index = int(resource_value.get('next_hop_index', 0)) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) - str_data = json.dumps({ - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': True}, - 'static_routes': {'static': [{ - 'prefix': prefix, - 'config': {'prefix': prefix}, - 'next_hops': { - 'next-hop': [{ - 'index': next_hop_index, - 'config': {'index': next_hop_index, 'next_hop': next_hop} - }] - } - }]} - }) + #str_data = json.dumps({ + # 'identifier': identifier, 'name': name, + # 'config': {'identifier': identifier, 'name': name, 'enabled': True}, + # 'static_routes': {'static': [{ + # 'prefix': prefix, + # 'config': {'prefix': prefix}, + # 'next_hops': { + # 'next-hop': [{ + # 'index': next_hop_index, + # 'config': {'index': next_hop_index, 'next_hop': next_hop} + # }] + # } + # }]} + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_prs : libyang.DContainer = yang_ni.create_path('protocols') + yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, name) + yang_ni_pr : libyang.DContainer = yang_ni_prs.create_path(yang_ni_pr_path) + yang_ni_pr.create_path('config/identifier', identifier) + yang_ni_pr.create_path('config/name', name ) + yang_ni_pr.create_path('config/enabled', True ) + + yang_ni_pr_srs : libyang.DContainer = yang_ni_pr.create_path('static-routes') + yang_ni_pr_sr_path = 'static[prefix="{:s}"]'.format(prefix) + yang_ni_pr_sr : libyang.DContainer = yang_ni_pr_srs.create_path(yang_ni_pr_sr_path) + yang_ni_pr_sr.create_path('config/prefix', prefix) + + yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') + yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) + yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) + yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) + + str_data = yang_ni_pr.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:protocol'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/tests/gnmi_openconfig/__init__.py b/src/device/tests/gnmi_openconfig/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/request_composers.py new file mode 100644 index 000000000..faa8425c8 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/request_composers.py @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = { + 'name': ni_name, 'type': ni_type + } + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = { + 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + } + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = { + 'name': ni_name, 'if_name': if_name, 'sif_index': sif_index + } + return str_path, str_data diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py new file mode 100644 index 000000000..4271b002f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage.py @@ -0,0 +1,285 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest, re +from typing import Dict, List, Tuple + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### POPULATE INTERFACE STORAGE ####################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + + +##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = sorted(resource_value.get('members')) + continue + + +##### GET EXPECTED INTERFACE CONFIG #################################################################################### + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + + +##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..69b7a609a --- /dev/null +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -0,0 +1,556 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route +from .storage import ( # pylint: disable=unused-import + storage, # be careful, order of symbols is important here!; storage should be the first one + get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, + populate_network_instances_storage +) + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + + +##### DRIVER FIXTURE ################################################################################################### + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + + +##### NETWORK INSTANCE DETAILS ######################################################################################### + +NI_NAME = 'test-l3-svc' +NI_TYPE = 'L3VRF' +NI_INTERFACES = [ + # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled + ('Ethernet1', 0, '192.168.1.1', 24, True), + ('Ethernet10', 0, '192.168.10.1', 24, True), +] +NI_STATIC_ROUTES = [ + # prefix, gateway, metric + ('172.0.0.0/24', '172.16.0.2', 1), + ('172.2.0.0/24', '172.16.0.3', 1), +] + + +##### TEST METHODS ##################################################################################################### + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance(NI_NAME, NI_TYPE), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + #for resource_key, resource_value in expected_getconfig: + # if resource_key == '/network_instance[default]/vlan[1]': + # resource_value['members'] = list() + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) + for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted([ + if_name + for if_name, _, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_add_interfaces_to_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_set_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) + for prefix, gateway, metric in NI_STATIC_ROUTES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + + + + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + # + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), + network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_deleteconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + #expected_getconfig = get_expected_interface_config(storage) + + #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + #num_diffs = len(diff_data) + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces_from_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interface_ids = sorted([ + '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ]) + results = set(results_deleteconfig) + assert len(results) == len(interface_ids) + for interface_id in interface_ids: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance(NI_NAME, 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py deleted file mode 100644 index 7d33d1a71..000000000 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ /dev/null @@ -1,637 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import deepdiff, logging, os, pytest, re, time -from typing import Dict, List, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' - -# pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -DRIVER_SETTING_ADDRESS = '172.20.20.101' -DRIVER_SETTING_PORT = 6030 -DRIVER_SETTING_USERNAME = 'admin' -DRIVER_SETTING_PASSWORD = 'admin' -DRIVER_SETTING_USE_TLS = False - -@pytest.fixture(scope='session') -def driver() -> GnmiOpenConfigDriver: - _driver = GnmiOpenConfigDriver( - DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, - username=DRIVER_SETTING_USERNAME, - password=DRIVER_SETTING_PASSWORD, - use_tls=DRIVER_SETTING_USE_TLS, - ) - _driver.Connect() - yield _driver - time.sleep(1) - _driver.Disconnect() - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### STORAGE POPULATORS ############################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### EXPECTED CONFIG COMPOSERS ######################################################################################## - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config - - -##### REQUEST COMPOSERS ################################################################################################ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = { - 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix - } - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def test_get_endpoints( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[test-l3-svc]', { - 'name': 'test-l3-svc', 'type': 'L3VRF' - }), - ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - for resource_key, resource_value in expected_getconfig: - if resource_key == '/network_instance[default]/vlan[1]': - resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - -#def test_unitary_gnmi_openconfig( -# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -#) -> None: -# #resources_to_get = [] -# resources_to_get = [RESOURCE_ENDPOINTS] -# #resources_to_get = [RESOURCE_INTERFACES] -# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] -# #resources_to_get = [RESOURCE_ROUTING_POLICIES] -# #resources_to_get = [RESOURCE_SERVICES] -# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) -# results_getconfig = driver.GetConfig(resources_to_get) -# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) -# -# #resources_to_set = [ -# # network_instance('test-svc', 'L3VRF'), -# # -# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/1', 0), -# # -# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/2', 0), -# # -# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), -# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), -# #] -# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) -# #results_setconfig = driver.SetConfig(resources_to_set) -# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) -# -# #resources_to_delete = [ -# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), -# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), -# # -# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), -# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), -# # -# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), -# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), -# # -# # #network_instance('20f66fb5', 'L3VRF'), -# #] -# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) -# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) -# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From 74a75434c591e0a4a74d94c64ea61600b2cdc3e2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 17:04:35 +0000 Subject: [PATCH 007/985] Device component - GNMI OpenConfig: - Corrected management of network instance interfaces - Improved unitary tests --- .../handlers/NetworkInstance.py | 12 ++ .../handlers/NetworkInstanceInterface.py | 8 +- .../test_unitary_gnmi_openconfig.py | 178 +++++++++++------- 3 files changed, 125 insertions(+), 73 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 1efed024c..b97612987 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -108,6 +108,18 @@ class NetworkInstanceHandler(_Handler): entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) entries.append((entry_net_inst_key, _net_inst)) + ni_interfaces = network_instance.get('interfaces', {}).get('interface', []) + for ni_interface in ni_interfaces: + #ni_if_id = ni_interface['id'] + ni_if_config = ni_interface['config'] + ni_if_name = ni_if_config['interface'] + ni_sif_index = ni_if_config['subinterface'] + ni_if_id = '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + + _interface = {'name': ni_name, 'id': ni_if_id, 'if_name': ni_if_name, 'sif_index': ni_sif_index} + entry_interface_key = '{:s}/interface[{:s}]'.format(entry_net_inst_key, ni_if_id) + entries.append((entry_interface_key, _interface)) + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) for ni_protocol in ni_protocols: ni_protocol_id = ni_protocol['identifier'].split(':')[-1] diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index ab105c2b0..af2178fe9 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -32,11 +32,7 @@ class NetworkInstanceInterfaceHandler(_Handler): ni_name = get_str(resource_value, 'name' ) # test-svc if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 sif_index = get_int(resource_value, 'sif_index', 0) # 0 - - if IS_CEOS: - if_id = if_name - else: - if_id = '{:s}.{:d}'.format(if_name, sif_index) + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -44,6 +40,8 @@ class NetworkInstanceInterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data + if IS_CEOS: if_id = if_name + str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) #str_data = json.dumps({ # 'id': if_id, diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 69b7a609a..47c8e1cdb 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -141,7 +141,6 @@ def test_get_network_instances( num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 - raise Exception() def test_set_network_instances( @@ -284,39 +283,80 @@ def test_add_interfaces_to_network_instance( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + interfaces = sorted([ + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_set_network_instance_static_routes( @@ -336,43 +376,45 @@ def test_set_network_instance_static_routes( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + prefixes = sorted([ + prefix + for prefix, _, _ in NI_STATIC_ROUTES + ]) + results = set(results_setconfig) + assert len(results) == len(prefixes) + for prefix in prefixes: + assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { + 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric + }) + for prefix, gateway, metric in NI_STATIC_ROUTES + ]) + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 - # - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_del_network_instance_static_routes( @@ -429,16 +471,16 @@ def test_del_interfaces_from_network_instance( LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) resources_to_delete = [ - network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) interface_ids = sorted([ - '{:s}.{:d}'.format(ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ]) results = set(results_deleteconfig) assert len(results) == len(interface_ids) -- GitLab From 03f748da04512622f43bb3c0551737434a25ec72 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 10 Jan 2024 18:36:50 +0000 Subject: [PATCH 008/985] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/storage.py | 285 -------- .../tests/gnmi_openconfig/storage/Storage.py | 23 + .../storage/StorageEndpoints.py | 72 ++ .../storage/StorageInterface.py | 122 ++++ .../storage/StorageNetworkInstance.py | 194 ++++++ .../tests/gnmi_openconfig/storage/Tools.py | 32 + .../tests/gnmi_openconfig/storage/__init__.py | 14 + .../test_unitary_gnmi_openconfig.py | 656 ++++++------------ .../tests/gnmi_openconfig/tools/__init__.py | 14 + .../gnmi_openconfig/tools/check_config.py | 82 +++ .../gnmi_openconfig/tools/check_updates.py | 21 + .../tools/expected_config_composers.py | 58 ++ .../{ => tools}/request_composers.py | 0 .../tools/result_config_adapters.py | 29 + 14 files changed, 864 insertions(+), 738 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageInterface.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Tools.py create mode 100644 src/device/tests/gnmi_openconfig/storage/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_config.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_updates.py create mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py rename src/device/tests/gnmi_openconfig/{ => tools}/request_composers.py (100%) create mode 100644 src/device/tests/gnmi_openconfig/tools/result_config_adapters.py diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py deleted file mode 100644 index 4271b002f..000000000 --- a/src/device/tests/gnmi_openconfig/storage.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest, re -from typing import Dict, List, Tuple - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### POPULATE INTERFACE STORAGE ####################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - - -##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### GET EXPECTED INTERFACE CONFIG #################################################################################### - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - - -##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/storage/Storage.py b/src/device/tests/gnmi_openconfig/storage/Storage.py new file mode 100644 index 000000000..4aaf29c99 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Storage.py @@ -0,0 +1,23 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .StorageEndpoints import StorageEndpoints +from .StorageInterface import StorageInterface +from .StorageNetworkInstance import StorageNetworkInstance + +class Storage: + def __init__(self) -> None: + self.endpoints = StorageEndpoints() + self.interfaces = StorageInterface() + self.network_instances = StorageNetworkInstance() diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py new file mode 100644 index 000000000..815a1b0ad --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -0,0 +1,72 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +RE_RESKEY_ENDPOINT = re.compile(r'^\/endpoints\/endpoint\[([^\]]+)\]$') + +ENDPOINT_PACKET_SAMPLE_TYPES : Dict[int, str] = { + 101: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-pkts', + 102: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-pkts', + 201: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-octets', + 202: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-octets', +} + +class Endpoints: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/endpoints/endpoint[{:s}]', ['uuid', 'type', 'sample_types']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ep_uuid : str, resource_value : Dict) -> None: + item = self._items.setdefault(ep_uuid, dict()) + item['uuid'] = ep_uuid + + for _, field_names in Endpoints.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + item['sample_types'] = { + sample_type_id : sample_type_path.format(ep_uuid) + for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() + } + + def remove(self, ep_uuid : str) -> None: + self._items.pop(ep_uuid, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Endpoints.STRUCT) + +class StorageEndpoints: + def __init__(self) -> None: + self.endpoints = Endpoints() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_ENDPOINT.match(resource_key) + if match is not None: + self.endpoints.add(match.group(1), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.endpoints.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py new file mode 100644 index 000000000..a0391e92f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/interface\[([^\]]+)\]' +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'$') +RE_RESKEY_ETHERNET = re.compile(PREFIX + r'\/ethernet$') +RE_RESKEY_SUBINTERFACE = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]$') +RE_RESKEY_IPV4_ADDRESS = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$') + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]', ['name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', + 'hardware-port', 'transceiver']), + ('/interface[{:s}]/ethernet', ['port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, if_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(if_name, dict()) + item['name'] = if_name + for _, field_names in Interfaces.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + def remove(self, if_name : str) -> None: + self._items.pop(if_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class SubInterfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]', ['index']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, if_name : str, subif_index : int) -> None: + item = self._items.setdefault((if_name, subif_index), dict()) + item['index'] = subif_index + + def remove(self, if_name : str, subif_index : int) -> None: + self._items.pop((if_name, subif_index), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, SubInterfaces.STRUCT) + +class IPv4Addresses: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]', ['ip', 'origin', 'prefix']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int, str], Dict] = dict() + + def add(self, if_name : str, subif_index : int, ipv4_address : str, resource_value : Dict) -> None: + item = self._items.setdefault((if_name, subif_index, ipv4_address), dict()) + item['ip' ] = ipv4_address + item['origin'] = resource_value.get('origin') + item['prefix'] = resource_value.get('prefix') + + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: + self._items.pop((if_name, subif_index, ipv4_address), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, IPv4Addresses.STRUCT) + +class StorageInterface: + def __init__(self) -> None: + self.interfaces = Interfaces() + self.subinterfaces = SubInterfaces() + self.ipv4_addresses = IPv4Addresses() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_ETHERNET.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_SUBINTERFACE.match(resource_key) + if match is not None: + self.subinterfaces.add(match.group(1), int(match.group(2))) + continue + + match = RE_RESKEY_IPV4_ADDRESS.match(resource_key) + if match is not None: + self.ipv4_addresses.add(match.group(1), int(match.group(2)), match.group(3), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.subinterfaces.compose_resources()) + expected_config.extend(self.ipv4_addresses.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py new file mode 100644 index 000000000..558cc032c --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -0,0 +1,194 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/network\_instance\[([^\]]+)\]' +RE_RESKEY_NET_INST = re.compile(PREFIX + r'$') +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'\/interface\[([^\]]+)\]$') +RE_RESKEY_PROTOCOL = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]$') +RE_RESKEY_PROTO_STATIC = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$') +RE_RESKEY_TABLE = re.compile(PREFIX + r'\/table\[([^\,]+)\,([^\]]+)\]$') +RE_RESKEY_VLAN = re.compile(PREFIX + r'\/vlan\[([^\]]+)\]$') + +class NetworkInstances: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]', ['name', 'type']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ni_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(ni_name, dict()) + item['name'] = ni_name + item['type'] = resource_value.get('type') + + def remove(self, ni_name : str) -> None: + self._items.pop(ni_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, NetworkInstances.STRUCT) + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, if_name : str) -> None: + item = self._items.setdefault((ni_name, if_name), dict()) + item['ni_name'] = ni_name + item['if_name'] = if_name + + def remove(self, ni_name : str, if_name : str) -> None: + self._items.pop((ni_name, if_name), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class Protocols: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]', ['id', 'name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str) -> None: + item = self._items.setdefault((ni_name, protocol), dict()) + item['id' ] = protocol + item['name'] = protocol + + def remove(self, ni_name : str, protocol : str) -> None: + self._items.pop((ni_name, protocol), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Protocols.STRUCT) + +class StaticRoutes: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, protocol, prefix), dict()) + item['prefix' ] = prefix + item['next_hops'] = sorted(resource_value.get('next_hops')) + + def remove(self, ni_name : str, protocol : str, prefix : str) -> None: + self._items.pop((ni_name, protocol, prefix), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, StaticRoutes.STRUCT) + +class Tables: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/table[{:s},{:s}]', ['protocol', 'address_family']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, address_family : str) -> None: + item = self._items.setdefault((ni_name, protocol, address_family), dict()) + item['protocol' ] = protocol + item['address_family'] = address_family + + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: + self._items.pop((ni_name, protocol, address_family), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Tables.STRUCT) + +class Vlans: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/vlan[{:d}]', ['vlan_id', 'name', 'members']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, ni_name : str, vlan_id : int, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, vlan_id), dict()) + item['vlan_id'] = vlan_id + item['name' ] = resource_value.get('name') + item['members'] = sorted(resource_value.get('members')) + + def remove(self, ni_name : str, vlan_id : int) -> None: + self._items.pop((ni_name, vlan_id), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Vlans.STRUCT) + +class StorageNetworkInstance: + def __init__(self) -> None: + self.network_instances = NetworkInstances() + self.interfaces = Interfaces() + self.protocols = Protocols() + self.protocol_static = StaticRoutes() + self.tables = Tables() + self.vlans = Vlans() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_NET_INST.match(resource_key) + if match is not None: + self.network_instances.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTOCOL.match(resource_key) + if match is not None: + self.protocols.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTO_STATIC.match(resource_key) + if match is not None: + self.protocol_static.add(match.group(1), match.group(2), match.group(3), resource_value) + continue + + match = RE_RESKEY_TABLE.match(resource_key) + if match is not None: + self.tables.add(match.group(1), match.group(2), match.group(3)) + continue + + match = RE_RESKEY_VLAN.match(resource_key) + if match is not None: + self.vlans.add(match.group(1), int(match.group(2)), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.network_instances.compose_resources()) + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.protocols.compose_resources()) + expected_config.extend(self.protocol_static.compose_resources()) + expected_config.extend(self.tables.compose_resources()) + expected_config.extend(self.vlans.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py new file mode 100644 index 000000000..4da48af46 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -0,0 +1,32 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Tuple + +def compose_resources( + storage : Dict[Tuple, Dict], config_struct : List[Tuple[str, List[str]]] +) -> List[Dict]: + expected_config = list() + + for resource_key_fields, resource_value_data in storage.items(): + for resource_key_template, resource_key_field_names in config_struct: + resource_key = resource_key_template.format(*resource_key_fields) + resource_value = { + field_name : resource_value_data[field_name] + for field_name in resource_key_field_names + if field_name in resource_value_data and resource_value_data[field_name] is not None + } + expected_config.append((resource_key, resource_value)) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/__init__.py b/src/device/tests/gnmi_openconfig/storage/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 47c8e1cdb..dd0561a2b 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -12,22 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import deepdiff, logging, os, pytest, re, time -from typing import Dict +import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position +import itertools, logging, pytest, time +from typing import Dict from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES ) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route -from .storage import ( # pylint: disable=unused-import - storage, # be careful, order of symbols is important here!; storage should be the first one - get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, - populate_network_instances_storage +from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.check_updates import check_updates +from .tools.expected_config_composers import ( + compose_expected_config__interface, compose_expected_config__network_instance ) +from .tools.request_composers import ( + interface, network_instance, network_instance_interface, network_instance_static_route +) +from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -56,19 +59,40 @@ def driver() -> GnmiOpenConfigDriver: _driver.Disconnect() +##### STORAGE FIXTURE ################################################################################################## + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield Storage() + + ##### NETWORK INSTANCE DETAILS ######################################################################################### -NI_NAME = 'test-l3-svc' -NI_TYPE = 'L3VRF' -NI_INTERFACES = [ - # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled - ('Ethernet1', 0, '192.168.1.1', 24, True), - ('Ethernet10', 0, '192.168.10.1', 24, True), -] -NI_STATIC_ROUTES = [ - # prefix, gateway, metric - ('172.0.0.0/24', '172.16.0.2', 1), - ('172.2.0.0/24', '172.16.0.3', 1), +NETWORK_INSTANCES = [ + { + 'name': 'test-l3-svc', + 'type': 'L3VRF', + 'interfaces': [ + {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + }, + { + 'name': 'test-l2-svc', + 'type': 'L2VSI', + 'interfaces': [ + {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + } ] @@ -76,523 +100,249 @@ NI_STATIC_ROUTES = [ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_endpoints(driver, storage) + storage.endpoints.populate(results_getconfig) + check_config_endpoints(driver, storage) def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_interfaces(driver, storage) + storage.interfaces.populate(results_getconfig) + check_config_interfaces(driver, storage) def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_network_instances(driver, storage) + storage.network_instances.populate(results_getconfig) + check_config_network_instances(driver, storage) def test_set_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance(NI_NAME, NI_TYPE), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_set.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - network_instances = sorted([NI_NAME]) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - #for resource_key, resource_value in expected_getconfig: - # if resource_key == '/network_instance[default]/vlan[1]': - # resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) - for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - interfaces = sorted([ - if_name - for if_name, _, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - interfaces = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) - for prefix, gateway, metric in NI_STATIC_ROUTES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_set = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - prefixes = sorted([ - prefix - for prefix, _, _ in NI_STATIC_ROUTES - ]) - results = set(results_setconfig) - assert len(results) == len(prefixes) - for prefix in prefixes: - assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { - 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric - }) - for prefix, gateway, metric in NI_STATIC_ROUTES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), - network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_deleteconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #expected_getconfig = get_expected_interface_config(storage) - - #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - #num_diffs = len(diff_data) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ + [ + (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ]))) - interface_ids = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_deleteconfig) - assert len(results) == len(interface_ids) - for interface_id in interface_ids: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ + [ni_if['name'] for ni_if in ni.get('interfaces', list())] + for ni in NETWORK_INSTANCES + ]))) - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure resources_to_delete = [ - network_instance(NI_NAME, 'L3VRF'), + network_instance(ni['name'], ni['type']) + for ni in NETWORK_INSTANCES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) - network_instances = sorted([NI_NAME]) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/__init__.py b/src/device/tests/gnmi_openconfig/tools/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py new file mode 100644 index 000000000..017a7038e --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -0,0 +1,82 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, deepdiff, logging, time +from typing import Callable, Dict, List, Tuple +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver +from device.tests.gnmi_openconfig.storage.Storage import Storage +from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_network_instance + +LOGGER = logging.getLogger(__name__) + +def check_expected_config( + driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + num_retry = 0 + return_data = None + while num_retry < max_retries: + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + return_data = copy.deepcopy(results_getconfig) + + results_getconfig = [ + func_adapt_returned_config(resource_key, resource_value) + for resource_key, resource_value in results_getconfig + ] + + diff_data = deepdiff.DeepDiff(sorted(expected_config), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(retry_delay) + num_retry -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + return return_data + +def check_config_endpoints( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_ENDPOINTS], storage.endpoints.get_expected_config(), + adapt_endpoint, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_interfaces( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_INTERFACES], storage.interfaces.get_expected_config(), + adapt_interface, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_network_instances( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + expected_config = + return check_expected_config( + driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), + adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay + ) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py new file mode 100644 index 000000000..7f31844cf --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -0,0 +1,21 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple + +def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: + results = set(results) + assert len(results) == len(item_ids) + for item_id in item_ids: + assert (format_str.format(*item_id), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py new file mode 100644 index 000000000..487476c01 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py @@ -0,0 +1,58 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + + +def compose_expected_config__network_instance( + network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False +) -> List[Dict]: + expected_config = list() + for network_instance in network_instances: + ni_name = network_instance['name'] + ni_type = network_instance['type'] + + expected_config.extend([ + ('/network_instance[{:s}]'.format(ni_name), { + 'name': ni_name, 'type': ni_type + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + + if include_interfaces: + expected_config.extend([ + ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { + + }) + for interface in network_instance.get('interfaces', list()) + ]) + + if include_static_routes: + expected_config.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + 'next_hop_index': 0, 'metric': static_route['metric'] + }) + for static_route in network_instance.get('static_routes', list()) + ]) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py similarity index 100% rename from src/device/tests/gnmi_openconfig/request_composers.py rename to src/device/tests/gnmi_openconfig/tools/request_composers.py diff --git a/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py new file mode 100644 index 000000000..3712f9365 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py @@ -0,0 +1,29 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, Tuple + +def adapt_endpoint(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_interface(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_network_instance(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + return resource_key, resource_value -- GitLab From 3a2f62eb760354066323ce060c835ed43e1363a2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 15:16:54 +0000 Subject: [PATCH 009/985] Device component - GNMI OpenConfig: - Updated unitary tests --- .../storage/StorageNetworkInstance.py | 11 +- .../test_unitary_gnmi_openconfig.py | 139 +++++++++--------- .../gnmi_openconfig/tools/check_config.py | 1 - .../tools/expected_config_composers.py | 58 -------- 4 files changed, 78 insertions(+), 131 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index 558cc032c..fa3364883 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -45,16 +45,14 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ('/network_instance[{:s}]/interface[{:s}]', []), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() def add(self, ni_name : str, if_name : str) -> None: - item = self._items.setdefault((ni_name, if_name), dict()) - item['ni_name'] = ni_name - item['if_name'] = if_name + self._items.setdefault((ni_name, if_name), dict()) def remove(self, ni_name : str, if_name : str) -> None: self._items.pop((ni_name, if_name), None) @@ -86,6 +84,11 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] + #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + # 'next_hop_index': 0, 'metric': static_route['metric'] + #}) + def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index dd0561a2b..3970e65a6 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -16,21 +16,17 @@ import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position -import itertools, logging, pytest, time +import logging, pytest, time from typing import Dict -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver -from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances -from .tools.check_updates import check_updates -from .tools.expected_config_composers import ( - compose_expected_config__interface, compose_expected_config__network_instance +from .storage.Storage import Storage +from .tools.check_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances ) +from .tools.check_updates import check_updates from .tools.request_composers import ( interface, network_instance, network_instance_interface, network_instance_static_route ) -from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -200,7 +196,7 @@ def test_add_interfaces_to_network_instance( subif_index = ni_if['sif_index'] resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) @@ -218,22 +214,26 @@ def test_set_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_set = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_set.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { + 'prefix': ni_sr_prefix, + }) - resources_to_set = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -246,22 +246,24 @@ def test_del_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_delete.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -274,26 +276,22 @@ def test_del_interfaces_from_network_instance( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ - [ - (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ]))) - + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -305,22 +303,24 @@ def test_del_interfaces( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - resources_to_delete = list(itertools.chain(*[ - [ - interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ - [ni_if['name'] for ni_if in ni.get('interfaces', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -333,16 +333,19 @@ def test_del_network_instances( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_delete.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.remove(ni_name) - resources_to_delete = [ - network_instance(ni['name'], ni['type']) - for ni in NETWORK_INSTANCES - ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) + check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 017a7038e..974acdeba 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -75,7 +75,6 @@ def check_config_network_instances( driver : GnmiOpenConfigDriver, storage : Storage, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: - expected_config = return check_expected_config( driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py deleted file mode 100644 index 487476c01..000000000 --- a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, List - - -def compose_expected_config__network_instance( - network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False -) -> List[Dict]: - expected_config = list() - for network_instance in network_instances: - ni_name = network_instance['name'] - ni_type = network_instance['type'] - - expected_config.extend([ - ('/network_instance[{:s}]'.format(ni_name), { - 'name': ni_name, 'type': ni_type - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - - if include_interfaces: - expected_config.extend([ - ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { - - }) - for interface in network_instance.get('interfaces', list()) - ]) - - if include_static_routes: - expected_config.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - 'next_hop_index': 0, 'metric': static_route['metric'] - }) - for static_route in network_instance.get('static_routes', list()) - ]) - - return expected_config -- GitLab From ffe63af338f70a638ec5df86c47be131ce117261 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 18:14:24 +0000 Subject: [PATCH 010/985] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/tools/check_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 974acdeba..5258da80c 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -25,10 +25,12 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) def check_expected_config( - driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: + LOGGER.info('expected_config = {:s}'.format(str(expected_config))) + num_retry = 0 return_data = None while num_retry < max_retries: -- GitLab From fce71b000b701d2d4e511092389eccb448d28987 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 12 Jan 2024 19:03:48 +0000 Subject: [PATCH 011/985] Device component - GNMI OpenConfig: - Corrected Component PORT retrieval - Corrected Static Route management - Corrected unitary tests - Added libyang to Dockerfile and corrected copy of clients - Corrected requirements.in --- src/device/Dockerfile | 21 +- src/device/requirements.in | 14 +- .../gnmi_openconfig/handlers/Component.py | 4 +- .../handlers/NetworkInstance.py | 17 +- .../handlers/NetworkInstanceStaticRoute.py | 6 +- .../storage/StorageEndpoints.py | 5 +- .../storage/StorageInterface.py | 11 +- .../storage/StorageNetworkInstance.py | 45 ++- .../tests/gnmi_openconfig/storage/Tools.py | 1 + .../test_unitary_gnmi_openconfig.py | 259 +++++++++--------- .../gnmi_openconfig/tools/check_updates.py | 5 +- .../{check_config.py => manage_config.py} | 30 +- .../tools/request_composers.py | 4 +- 13 files changed, 250 insertions(+), 172 deletions(-) rename src/device/tests/gnmi_openconfig/tools/{check_config.py => manage_config.py} (71%) diff --git a/src/device/Dockerfile b/src/device/Dockerfile index 656662552..2bcb5322a 100644 --- a/src/device/Dockerfile +++ b/src/device/Dockerfile @@ -53,6 +53,21 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install build-essential cmake libpcre2-dev python3-dev python3-cffi && \ + rm -rf /var/lib/apt/lists/* +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/device WORKDIR /var/teraflow/device @@ -62,9 +77,11 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ COPY src/device/. device/ -COPY src/monitoring/. monitoring/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ # Start the service ENTRYPOINT ["python", "-m", "device.service"] diff --git a/src/device/requirements.in b/src/device/requirements.in index d8a33455e..20ed1e2dc 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -12,30 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. - anytree==2.8.0 APScheduler==3.10.1 bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 +ipaddress Jinja2==3.0.3 +libyang==2.8.0 +macaddress ncclient==0.6.13 p4runtime==1.3.0 pandas==1.5.* paramiko==2.9.2 +pyang==2.6.* +git+https://github.com/robshakir/pyangbind.git python-json-logger==2.0.2 #pytz==2021.3 #redis==4.1.2 requests==2.27.1 requests-mock==1.9.3 -xmltodict==0.12.0 tabulate -ipaddress -macaddress -yattag -pyang==2.6.* -git+https://github.com/robshakir/pyangbind.git websockets==10.4 +xmltodict==0.12.0 +yattag # pip's dependency resolver does not take into account installed packages. # p4runtime does not specify the version of grpcio/protobuf it needs, so it tries to install latest one diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 73728192f..9b92c6b83 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging # libyang +import json, logging, re # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType from ._Handler import _Handler @@ -55,7 +55,7 @@ class ComponentHandler(_Handler): # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. - interface_name = component_name.lower().replace('-port', '') + interface_name = re.sub(r'\-[pP][oO][rR][tT]', '', component_name) endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index b97612987..d8231b2a6 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -134,18 +134,15 @@ class NetworkInstanceHandler(_Handler): for static_route in static_routes: static_route_prefix = static_route['prefix'] - next_hops = static_route.get('next-hops', {}).get('next-hop', []) - _next_hops = [ - { - 'index' : next_hop['index'], - 'gateway': next_hop['config']['next-hop'], - 'metric' : next_hop['config']['metric'], + next_hops = { + next_hop['index'] : { + 'next_hop': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], } - for next_hop in next_hops - ] - _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + for next_hop in static_route.get('next-hops', {}).get('next-hop', []) + } - _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + _static_route = {'prefix': static_route_prefix, 'next_hops': next_hops} entry_static_route_key = '{:s}/static_routes[{:s}]'.format( entry_protocol_key, static_route_prefix ) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 0343e3cba..03c04e316 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -39,8 +39,8 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' - next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_str(resource_value, 'next_hop_index') # AUTO_1_172-0-0-1 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) @@ -74,7 +74,7 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_ni_pr_sr.create_path('config/prefix', prefix) yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') - yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh_path = 'next-hop[index="{:s}"]'.format(next_hop_index) yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py index 815a1b0ad..d2596b732 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -39,13 +39,16 @@ class Endpoints: for _, field_names in Endpoints.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) item['sample_types'] = { sample_type_id : sample_type_path.format(ep_uuid) for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() } + def get(self, ep_uuid : str) -> Dict: + return self._items.get(ep_uuid) + def remove(self, ep_uuid : str) -> None: self._items.pop(ep_uuid, None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py index a0391e92f..0933433cb 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -37,7 +37,10 @@ class Interfaces: item['name'] = if_name for _, field_names in Interfaces.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) + + def get(self, if_name : str) -> Dict: + return self._items.get(if_name) def remove(self, if_name : str) -> None: self._items.pop(if_name, None) @@ -57,6 +60,9 @@ class SubInterfaces: item = self._items.setdefault((if_name, subif_index), dict()) item['index'] = subif_index + def get(self, if_name : str, subif_index : int) -> Dict: + return self._items.get((if_name, subif_index)) + def remove(self, if_name : str, subif_index : int) -> None: self._items.pop((if_name, subif_index), None) @@ -77,6 +83,9 @@ class IPv4Addresses: item['origin'] = resource_value.get('origin') item['prefix'] = resource_value.get('prefix') + def get(self, if_name : str, subif_index : int, ipv4_address : str) -> Dict: + return self._items.get((if_name, subif_index, ipv4_address)) + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: self._items.pop((if_name, subif_index, ipv4_address), None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index fa3364883..ba437ef9d 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -37,6 +37,9 @@ class NetworkInstances: item['name'] = ni_name item['type'] = resource_value.get('type') + def get(self, ni_name : str) -> Dict: + return self._items.get(ni_name) + def remove(self, ni_name : str) -> None: self._items.pop(ni_name, None) @@ -45,17 +48,24 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', []), + ('/network_instance[{:s}]/interface[{:s}.{:d}]', ['name', 'id', 'if_name', 'sif_index']), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() - def add(self, ni_name : str, if_name : str) -> None: - self._items.setdefault((ni_name, if_name), dict()) + def add(self, ni_name : str, if_name : str, sif_index : int) -> None: + item = self._items.setdefault((ni_name, if_name, sif_index), dict()) + item['name' ] = ni_name + item['id' ] = '{:s}.{:d}'.format(if_name, sif_index) + item['if_name' ] = if_name + item['sif_index'] = sif_index + + def get(self, ni_name : str, if_name : str, sif_index : int) -> Dict: + return self._items.get((ni_name, if_name, sif_index)) - def remove(self, ni_name : str, if_name : str) -> None: - self._items.pop((ni_name, if_name), None) + def remove(self, ni_name : str, if_name : str, sif_index : int) -> None: + self._items.pop((ni_name, if_name, sif_index), None) def compose_resources(self) -> List[Dict]: return compose_resources(self._items, Interfaces.STRUCT) @@ -73,6 +83,9 @@ class Protocols: item['id' ] = protocol item['name'] = protocol + def get(self, ni_name : str, protocol : str) -> Dict: + return self._items.get((ni_name, protocol)) + def remove(self, ni_name : str, protocol : str) -> None: self._items.pop((ni_name, protocol), None) @@ -84,18 +97,16 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] - #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - # 'next_hop_index': 0, 'metric': static_route['metric'] - #}) - def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: item = self._items.setdefault((ni_name, protocol, prefix), dict()) item['prefix' ] = prefix - item['next_hops'] = sorted(resource_value.get('next_hops')) + item['next_hops'] = resource_value.get('next_hops') + + def get(self, ni_name : str, protocol : str, prefix : str) -> Dict: + return self._items.get((ni_name, protocol, prefix)) def remove(self, ni_name : str, protocol : str, prefix : str) -> None: self._items.pop((ni_name, protocol, prefix), None) @@ -116,6 +127,9 @@ class Tables: item['protocol' ] = protocol item['address_family'] = address_family + def get(self, ni_name : str, protocol : str, address_family : str) -> Dict: + return self._items.get((ni_name, protocol, address_family)) + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: self._items.pop((ni_name, protocol, address_family), None) @@ -136,6 +150,9 @@ class Vlans: item['name' ] = resource_value.get('name') item['members'] = sorted(resource_value.get('members')) + def get(self, ni_name : str, vlan_id : int) -> Dict: + return self._items.get((ni_name, vlan_id)) + def remove(self, ni_name : str, vlan_id : int) -> None: self._items.pop((ni_name, vlan_id), None) @@ -160,7 +177,11 @@ class StorageNetworkInstance: match = RE_RESKEY_INTERFACE.match(resource_key) if match is not None: - self.interfaces.add(match.group(1), match.group(2)) + if_id = match.group(2) + if_id_parts = if_id.split('.') + if_name = if_id_parts[0] + sif_index = 0 if len(if_id_parts) == 1 else int(if_id_parts[1]) + self.interfaces.add(match.group(1), if_name, sif_index) continue match = RE_RESKEY_PROTOCOL.match(resource_key) diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py index 4da48af46..c9dab12e6 100644 --- a/src/device/tests/gnmi_openconfig/storage/Tools.py +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -21,6 +21,7 @@ def compose_resources( for resource_key_fields, resource_value_data in storage.items(): for resource_key_template, resource_key_field_names in config_struct: + if isinstance(resource_key_fields, (str, int, float, bool)): resource_key_fields = (resource_key_fields,) resource_key = resource_key_template.format(*resource_key_fields) resource_value = { field_name : resource_value_data[field_name] diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 3970e65a6..a601e1f23 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -17,11 +17,15 @@ os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position import logging, pytest, time -from typing import Dict +from typing import Dict, List +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver from .storage.Storage import Storage -from .tools.check_config import ( - check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.manage_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances, del_config, get_config, set_config ) from .tools.check_updates import check_updates from .tools.request_composers import ( @@ -69,26 +73,26 @@ NETWORK_INSTANCES = [ 'name': 'test-l3-svc', 'type': 'L3VRF', 'interfaces': [ - {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet1', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, ], 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, ] }, - { - 'name': 'test-l2-svc', - 'type': 'L2VSI', - 'interfaces': [ - {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, - ], - 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, - ] - } + #{ + # 'name': 'test-l2-svc', + # 'type': 'L2VSI', + # 'interfaces': [ + # {'name': 'Ethernet2', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + # {'name': 'Ethernet4', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + # ], + # 'static_routes': [ + # {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + # {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, + # ] + #} ] @@ -98,7 +102,7 @@ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_endpoints(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_ENDPOINTS]) storage.endpoints.populate(results_getconfig) check_config_endpoints(driver, storage) @@ -107,7 +111,7 @@ def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_interfaces(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_INTERFACES]) storage.interfaces.populate(results_getconfig) check_config_interfaces(driver, storage) @@ -116,7 +120,7 @@ def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_network_instances(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_NETWORK_INSTANCES]) storage.network_instances.populate(results_getconfig) check_config_network_instances(driver, storage) @@ -136,17 +140,18 @@ def test_set_network_instances( resources_to_set.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) - - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + storage.network_instances.protocols.add(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') + + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_set_interfaces( +def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -154,33 +159,24 @@ def test_set_interfaces( check_config_network_instances(driver, storage) resources_to_set = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_set.append(interface( - if_name, subif_index, ipv4_address, ipv4_prefix, enabled - )) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { - 'origin' : 'STATIC', 'prefix': ipv4_prefix - }) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/interface[{:s}]', if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_add_interfaces_to_network_instance( +def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -188,23 +184,30 @@ def test_add_interfaces_to_network_instance( check_config_network_instances(driver, storage) resources_to_set = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name in default_vlan_members: default_vlan_members.remove(if_name) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_set_network_instance_static_routes( @@ -219,24 +222,28 @@ def test_set_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_set.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_set.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocols.add(ni_name, 'STATIC') storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { - 'prefix': ni_sr_prefix, + 'prefix': ni_sr_prefix, 'next_hops': { + ni_sr_next_hop_index: {'next_hop': ni_sr_next_hop, 'metric': ni_sr_metric} + } }) + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instance_static_routes( @@ -251,79 +258,80 @@ def test_del_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_delete.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_delete.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + + storage.network_instances.protocols.remove(ni_name, 'STATIC') storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces_from_network_instance( +def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name not in default_vlan_members: default_vlan_members.append(if_name) - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces( + +def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name, subif_index) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instances( @@ -331,7 +339,7 @@ def test_del_network_instances( storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() ni_names = list() @@ -341,11 +349,12 @@ def test_del_network_instances( resources_to_delete.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.remove(ni_name) + storage.network_instances.protocols.remove(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py index 7f31844cf..a9e2a1be9 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_updates.py +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -17,5 +17,6 @@ from typing import Iterable, List, Tuple def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: results = set(results) assert len(results) == len(item_ids) - for item_id in item_ids: - assert (format_str.format(*item_id), True) in results + for item_id_fields in item_ids: + if isinstance(item_id_fields, (str, int, float, bool)): item_id_fields = (item_id_fields,) + assert (format_str.format(*item_id_fields), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/manage_config.py similarity index 71% rename from src/device/tests/gnmi_openconfig/tools/check_config.py rename to src/device/tests/gnmi_openconfig/tools/manage_config.py index 5258da80c..72d6a09d3 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/manage_config.py @@ -13,7 +13,7 @@ # limitations under the License. import copy, deepdiff, logging, time -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, List, Tuple, Union from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES @@ -24,6 +24,28 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) +def get_config(driver : GnmiOpenConfigDriver, resources_to_get : List[str]) -> List[Tuple[str, Dict]]: + LOGGER.info('[get_config] resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('[get_config] results_getconfig = {:s}'.format(str(results_getconfig))) + return results_getconfig + +def set_config( + driver : GnmiOpenConfigDriver, resources_to_set : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('[set_config] resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('[set_config] results_setconfig = {:s}'.format(str(results_setconfig))) + return results_setconfig + +def del_config( + driver : GnmiOpenConfigDriver, resources_to_delete : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + return results_deleteconfig + def check_expected_config( driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, @@ -34,9 +56,7 @@ def check_expected_config( num_retry = 0 return_data = None while num_retry < max_retries: - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + results_getconfig = get_config(driver, resources_to_get) return_data = copy.deepcopy(results_getconfig) results_getconfig = [ @@ -49,7 +69,7 @@ def check_expected_config( if num_diffs == 0: break # let the device take some time to reconfigure time.sleep(retry_delay) - num_retry -= 1 + num_retry += 1 if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 diff --git a/src/device/tests/gnmi_openconfig/tools/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py index faa8425c8..be0587101 100644 --- a/src/device/tests/gnmi_openconfig/tools/request_composers.py +++ b/src/device/tests/gnmi_openconfig/tools/request_composers.py @@ -29,10 +29,10 @@ def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: } return str_path, str_data -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: +def network_instance_static_route(ni_name, prefix, next_hop_index, next_hop, metric=1) -> Tuple[str, Dict]: str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) str_data = { - 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + 'name': ni_name, 'prefix': prefix, 'next_hop_index': next_hop_index, 'next_hop': next_hop, 'metric': metric } return str_path, str_data -- GitLab From c6f443a5d825bfda0bdaa2a34db283d30a018da9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 15 Jan 2024 13:48:14 +0000 Subject: [PATCH 012/985] Device component - GNMI OpenConfig: - Added TODO.txt --- .../service/drivers/gnmi_openconfig/TODO.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/device/service/drivers/gnmi_openconfig/TODO.txt diff --git a/src/device/service/drivers/gnmi_openconfig/TODO.txt b/src/device/service/drivers/gnmi_openconfig/TODO.txt new file mode 100644 index 000000000..ba8ff1c2c --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/TODO.txt @@ -0,0 +1,15 @@ +- update parse() @ InterfaceCounter.py +- update compose() @ NetworkInstance.py +- update compose() @ NetworkInstanceInterface.py +- implement parse() @ NetworkInstanceInterface.py +- update compose() @ NetworkInstanceStaticRoute.py +- implement parse() @ NetworkInstanceStaticRoute.py +- Fix MonitoringThread.py + + +there is an error removing static routes that makes unitary tests to crash +uncomment commented check_config_network_instance and validate + +- implement L2 VPN with BGP +- implement L3 VPN with BGP +- test static routes with ping -- GitLab From 6958674bf5af23810e1e318981c979450683069c Mon Sep 17 00:00:00 2001 From: hajipour Date: Fri, 26 Jan 2024 14:49:05 +0100 Subject: [PATCH 013/985] bug fix: port & device uuids extracted from device protobuf message instead of custom feature which is dedicated for emulated devices. --- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index a78d28193..c80591992 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -14,6 +14,7 @@ import json import logging +import re import time from decimal import ROUND_HALF_EVEN, Decimal from flask.json import jsonify @@ -26,6 +27,9 @@ from common.tools.object_factory.Service import json_service_id LOGGER = logging.getLogger(__name__) +RE_PORT_AND_IP_MATCH = r'resource_key: \"\/interface\[(.*)\]\/subinterface\[.*\]\"[.|\n]*.*\\\"address_ip\\\": \\\"{}\\\"' +RE_ENDPOINT_UUID_PORT_MATCH = r'\s*\n?\s*endpoint_uuid .\s*\n?\s*uuid: \"(.*)\"\n?\s*.\n?\s*.\n?\s*name: \"{}\"' +RE_DEVICE_UUID_MATCH = r'\n?\s*device_uuid \{\s*\n?\s*uuid: \"(.*)\"\n?\s*\}\n?\s*\}' def service_2_bwInfo(service: Service) -> dict: response = {} @@ -75,17 +79,20 @@ def bwInfo_2_service(client, bwInfo: dict) -> Service: a_ip = bwInfo['sessionFilter'][0]['sourceIp'] z_ip = bwInfo['sessionFilter'][0]['dstAddress'] - devices = client.ListDevices(Empty()).devices + devices = (str(device) for device in client.ListDevices(Empty()).devices) for device in devices: - for cr in device.device_config.config_rules: - if cr.WhichOneof('config_rule') == 'custom' and cr.custom.resource_key == '_connect/settings': - for ep in json.loads(cr.custom.resource_value)['endpoints']: - if 'ip' in ep and (ep['ip'] == a_ip or ep['ip'] == z_ip): - ep_id = EndPointId() - ep_id.endpoint_uuid.uuid = ep['uuid'] - ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid - service.service_endpoint_ids.append(ep_id) - + for ep_id in (a_ip, z_ip): + if look_up := re.search(RE_PORT_AND_IP_MATCH.format(ep_id), device): + physical_port = look_up.groups(0)[0] + # 'PORT-' is added as a prefix + port = 'PORT-' + physical_port + port_uuid = re.search(RE_ENDPOINT_UUID_PORT_MATCH.format(port), device).group(0)[0] + device_uuid = re.search(RE_DEVICE_UUID_MATCH, device).group(0)[0] + ep_id = EndPointId() + ep_id.endpoint_uuid.uuid = port_uuid + ep_id.device_id.device_uuid.uuid = device_uuid + service.service_endpoint_ids.append(ep_id) + service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM if 'appInsId' in bwInfo: -- GitLab From 27b2104dd4fb4615db630ec424ee20258acbeec0 Mon Sep 17 00:00:00 2001 From: hajipour Date: Fri, 26 Jan 2024 23:16:03 +0100 Subject: [PATCH 014/985] refactoring: port lookup done sequentially in device protobuf instead of global regex search --- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index c80591992..17199315d 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -27,9 +27,8 @@ from common.tools.object_factory.Service import json_service_id LOGGER = logging.getLogger(__name__) -RE_PORT_AND_IP_MATCH = r'resource_key: \"\/interface\[(.*)\]\/subinterface\[.*\]\"[.|\n]*.*\\\"address_ip\\\": \\\"{}\\\"' -RE_ENDPOINT_UUID_PORT_MATCH = r'\s*\n?\s*endpoint_uuid .\s*\n?\s*uuid: \"(.*)\"\n?\s*.\n?\s*.\n?\s*name: \"{}\"' -RE_DEVICE_UUID_MATCH = r'\n?\s*device_uuid \{\s*\n?\s*uuid: \"(.*)\"\n?\s*\}\n?\s*\}' +RE_CONFIG_RULE_IF_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') +RE_CONFIG_RULE_ADDRESS_IP = re.compile(r'\\\"address_ip\\\": \\\"((?:[0-9]{1,3}\.){3}[0-9]{1,3})\\\"') def service_2_bwInfo(service: Service) -> dict: response = {} @@ -79,20 +78,20 @@ def bwInfo_2_service(client, bwInfo: dict) -> Service: a_ip = bwInfo['sessionFilter'][0]['sourceIp'] z_ip = bwInfo['sessionFilter'][0]['dstAddress'] - devices = (str(device) for device in client.ListDevices(Empty()).devices) + devices = client.ListDevices(Empty()).devices for device in devices: - for ep_id in (a_ip, z_ip): - if look_up := re.search(RE_PORT_AND_IP_MATCH.format(ep_id), device): - physical_port = look_up.groups(0)[0] - # 'PORT-' is added as a prefix - port = 'PORT-' + physical_port - port_uuid = re.search(RE_ENDPOINT_UUID_PORT_MATCH.format(port), device).group(0)[0] - device_uuid = re.search(RE_DEVICE_UUID_MATCH, device).group(0)[0] - ep_id = EndPointId() - ep_id.endpoint_uuid.uuid = port_uuid - ep_id.device_id.device_uuid.uuid = device_uuid - service.service_endpoint_ids.append(ep_id) - + device_endpoint_uuids = {ep.name:ep.endpoint_id.endpoint_uuid for ep in device.device_endpoints} + for cr in device.device_config.config_rules: + if cr.WhichOneof('config_rule') == 'custom': + match_subif = RE_CONFIG_RULE_IF_SUBIF.match(cr.custom.resource_key) + match_ip = RE_CONFIG_RULE_ADDRESS_IP.match(cr.custom.resource_value) + if match_subif and match_ip and match_ip.groups(0)[0] in [a_ip, z_ip]: + # `PORT-` added as prefix + port_name = 'PORT-' + match_subif.groups(0)[0] + ep_id = EndPointId() + ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] + ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid + service.service_endpoint_ids.append(ep_id) service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM if 'appInsId' in bwInfo: -- GitLab From c2179dd845b717806fb0acc522f80e70581e0a37 Mon Sep 17 00:00:00 2001 From: hajipour Date: Fri, 26 Jan 2024 23:44:16 +0100 Subject: [PATCH 015/985] bug fix: Showing service info error in webui when service is created by BWM resolved. --- src/webui/service/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index 05b2eeaf0..5211ad935 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -48,7 +48,7 @@ def json_to_list(json_str : str) -> List[Union[str, Tuple[str, str]]]: if isinstance(data, dict): return [('kv', (key, value)) for key, value in data.items()] - elif isinstance(data, list): + elif isinstance(data, list) and not isinstance(data[0], dict): return [('item', ', '.join(data))] else: return [('item', str(data))] -- GitLab From ccdf13d912422c550f141a4f6ca81533ff749aad Mon Sep 17 00:00:00 2001 From: hajipour Date: Sun, 28 Jan 2024 11:18:15 +0100 Subject: [PATCH 016/985] bug fix:device's uuid assigned to EndpointId.device_id.device_uuid.uuid in bwm service creation. --- src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index 17199315d..b9ed73850 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -90,7 +90,7 @@ def bwInfo_2_service(client, bwInfo: dict) -> Service: port_name = 'PORT-' + match_subif.groups(0)[0] ep_id = EndPointId() ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] - ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid + ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid service.service_endpoint_ids.append(ep_id) service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM -- GitLab From 4bbdf917ca1ea8a6a19a2ea9e5bfbe4b3ddf60dd Mon Sep 17 00:00:00 2001 From: hajipour Date: Sun, 28 Jan 2024 17:26:48 +0100 Subject: [PATCH 017/985] bug fix: json_to_list function bug when json_str is empty list resolved. --- src/webui/service/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index 5211ad935..7146f321b 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -45,10 +45,12 @@ def json_to_list(json_str : str) -> List[Union[str, Tuple[str, str]]]: data = json.loads(json_str) except: # pylint: disable=bare-except return [('item', str(json_str))] - + if isinstance(data, dict): return [('kv', (key, value)) for key, value in data.items()] - elif isinstance(data, list) and not isinstance(data[0], dict): + elif isinstance(data, list) and len(data) == 1 and isinstance(data[0], dict): # BWM use-case + return [('kv', (key, value)) for key, value in data[0].items()] + elif isinstance(data, list) and all(isinstance(d, str) for d in data): return [('item', ', '.join(data))] else: return [('item', str(data))] -- GitLab From f10241a6c1228d1100750f5e79bdff1e88fbce02 Mon Sep 17 00:00:00 2001 From: hajipour Date: Mon, 12 Feb 2024 12:24:01 +0000 Subject: [PATCH 018/985] bug fix: temporary fix for MEC PoC demo. Config rules and OpenConfig driver for L3VPN should change based on telefonica recipe. --- .../drivers/openconfig/OpenConfigDriver.py | 5 +- .../drivers/openconfig/templates/Tools.py | 3 +- .../VPN/Network_instance_multivendor.py | 36 ++++- .../nbi_plugins/etsi_bwm/Resources.py | 15 +- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 140 +++++++++++++----- .../algorithms/tools/ComposeConfigRules.py | 18 +++ .../l3nm_openconfig/ConfigRules.py | 66 +++++++-- 7 files changed, 229 insertions(+), 54 deletions(-) diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 8c6e07b3f..99ae1c8db 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -226,8 +226,11 @@ def edit_config( chk_length(str_resource_name, resource, min_length=2, max_length=2) resource_key,resource_value = resource chk_string(str_resource_name + '.key', resource_key, allow_empty=False) + str_config_messages = compose_config( # get template for configuration - resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) + resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer='pyangbind') + # str_config_messages = compose_config( # get template for configuration + # resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) for str_config_message in str_config_messages: # configuration of the received templates if str_config_message is None: raise UnsupportedResourceKeyException(resource_key) logger.debug('[{:s}] str_config_message[{:d}] = {:s}'.format( diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 79bebef51..3e8043680 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -61,7 +61,8 @@ def generate_templates(resource_key: str, resource_value: str, delete: bool,vend elif "inter_instance_policies" in resource_key: result_templates.append(associate_RP_to_NI(data)) elif "protocols" in resource_key: - if vendor == "ADVA": result_templates.append(add_protocol_NI(data, vendor, delete)) + result_templates.append(add_protocol_NI(data, vendor, delete)) + # if vendor == "ADVA": result_templates.append(add_protocol_NI(data, vendor, delete)) elif "table_connections" in resource_key: result_templates.append(create_table_conns(data, delete)) elif "interface" in resource_key: diff --git a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py index c4d494ea6..e36955a0d 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py @@ -116,6 +116,9 @@ def add_protocol_NI(parameters,vendor, DEL): else: with tag('network-instance'): with tag('name'):text(parameters['name']) + with tag('config'): + with tag('name'): text(parameters['name']) + with tag('type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"'): text('oc-ni-types:DEFAULT_INSTANCE') with tag('protocols'): with tag('protocol'): with tag('identifier', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) @@ -123,14 +126,41 @@ def add_protocol_NI(parameters,vendor, DEL): with tag('config'): with tag('identifier', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) with tag('name') :text(parameters['protocol_name']) + with tag('enabled'): text('true') if "BGP" in parameters['identifier']: with tag('bgp'): with tag('global'): + with tag('afi-safis'): + with tag('afi-safi'): + with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') + with tag('config'): + with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') + with tag('enabled'): text('true') with tag('config'): with tag('as') :text(parameters['as']) - if "router-id" in parameters: - with tag('router-id'):text(parameters['router-id']) - if vendor == "ADVA": + with tag('peer-groups'): + with tag('peer-group'): + with tag('peer-group-name'): text('IBGP') + with tag('config'): + with tag('peer-group-name'): text('IBGP') + with tag('peer-as'): text(parameters['protocol_name']) + with tag('afi-safis'): + with tag('afi-safi'): + with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') + with tag('config'): + with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') + with tag('enabled'): text('true') + + if 'neighbors' in parameters: + with tag('neighbors'): + for neighbor in parameters['neighbors']: + with tag('neighbor'): + with tag('neighbor-address'): text(neighbor['ip_address']) + with tag('config'): + with tag('neighbor-address'): text(neighbor['ip_address']) + with tag('peer-group'): text('IBGP') + # if vendor == "ADVA": + if True: with tag('tables'): with tag('table'): with tag('protocol', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py index 3fccbbb55..394b50de8 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py @@ -13,7 +13,9 @@ # limitations under the License. import copy, deepmerge, json, logging +from typing import Dict from common.Constants import DEFAULT_CONTEXT_NAME +from werkzeug.exceptions import UnsupportedMediaType from context.client.ContextClient import ContextClient from flask_restful import Resource, request from service.client.ServiceClient import ServiceClient @@ -37,15 +39,20 @@ class BwInfo(_Resource): return bw_allocations def post(self): - bwinfo = request.get_json() - service = bwInfo_2_service(self.client, bwinfo) + if not request.is_json: + raise UnsupportedMediaType('JSON payload is required') + request_data: Dict = request.get_json() + service = bwInfo_2_service(self.client, request_data) stripped_service = copy.deepcopy(service) stripped_service.ClearField('service_endpoint_ids') stripped_service.ClearField('service_constraints') stripped_service.ClearField('service_config') - response = format_grpc_to_json(self.service_client.CreateService(stripped_service)) - response = format_grpc_to_json(self.service_client.UpdateService(service)) + try: + response = format_grpc_to_json(self.service_client.CreateService(stripped_service)) + response = format_grpc_to_json(self.service_client.UpdateService(service)) + except Exception as e: # pylint: disable=broad-except + return e return response diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index a78d28193..d3be769c9 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -14,22 +14,35 @@ import json import logging +import re import time from decimal import ROUND_HALF_EVEN, Decimal from flask.json import jsonify from common.proto.context_pb2 import ( - ContextId, Empty, EndPointId, ServiceId, ServiceTypeEnum, Service, Constraint, Constraint_SLA_Capacity, + ContextId, Empty, EndPointId, ServiceId, ServiceTypeEnum, Service, ServiceStatusEnum, Constraint, Constraint_SLA_Capacity, ConfigRule, ConfigRule_Custom, ConfigActionEnum) from common.tools.grpc.Tools import grpc_message_to_json +from common.tools.grpc.ConfigRules import update_config_rule_custom from common.tools.object_factory.Context import json_context_id from common.tools.object_factory.Service import json_service_id LOGGER = logging.getLogger(__name__) +ENDPOINT_SETTINGS_KEY = '/device[{:s}]/endpoint[{:s}]/vlan[{:d}]/settings' +RE_CONFIG_RULE_IF_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') +MEC_CONSIDERED_FIELDS = ['requestType', 'sessionFilter', 'fixedAllocation', 'allocationDirection'] +ALLOCATION_DIRECTION_DESCRIPTIONS = { + '00' : 'Downlink (towards the UE)', + '01' : 'Uplink (towards the application/session)', + '10' : 'Symmetrical'} +VLAN_TAG = 0 +PREFIX_LENGTH = 24 +BGP_AS = 65000 +policy_AZ = 'srv_{:d}_a'.format(VLAN_TAG) +policy_ZA = 'srv_{:d}_b'.format(VLAN_TAG) def service_2_bwInfo(service: Service) -> dict: response = {} - # allocationDirection = '??' # String: 00 = Downlink (towards the UE); 01 = Uplink (towards the application/session); 10 = Symmetrical response['appInsId'] = service.service_id.service_uuid.uuid # String: Application instance identifier for constraint in service.service_constraints: if constraint.WhichOneof('constraint') == 'sla_capacity': @@ -55,47 +68,108 @@ def service_2_bwInfo(service: Service) -> dict: return response -def bwInfo_2_service(client, bwInfo: dict) -> Service: +def bwInfo_2_service(client, bw_info: dict) -> Service: + # add description to allocationDirection code + if ad_code := bw_info.get('allocationDirection'): + bw_info['allocationDirection'] = {'code': ad_code, 'description': ALLOCATION_DIRECTION_DESCRIPTIONS[ad_code]} + if 'sessionFilter' in bw_info: + bw_info['sessionFilter'] = bw_info['sessionFilter'][0] # Discard other items in sessionFilter field + service = Service() - for key in ['allocationDirection', 'fixedBWPriority', 'requestType', 'timeStamp', 'sessionFilter']: - if key not in bwInfo: - continue - config_rule = ConfigRule() - config_rule.action = ConfigActionEnum.CONFIGACTION_SET - config_rule_custom = ConfigRule_Custom() - config_rule_custom.resource_key = key - if key != 'sessionFilter': - config_rule_custom.resource_value = str(bwInfo[key]) - else: - config_rule_custom.resource_value = json.dumps(bwInfo[key]) - config_rule.custom.CopyFrom(config_rule_custom) - service.service_config.config_rules.append(config_rule) - - if 'sessionFilter' in bwInfo: - a_ip = bwInfo['sessionFilter'][0]['sourceIp'] - z_ip = bwInfo['sessionFilter'][0]['dstAddress'] + + service_config_rules = service.service_config.config_rules + + route_distinguisher = '{:5d}:{:03d}'.format(BGP_AS, VLAN_TAG) + settings_cr_key = '/settings' + settings_cr_value = {'bgp_as':(BGP_AS, True), 'route_distinguisher': (route_distinguisher, True)} + update_config_rule_custom(service_config_rules, settings_cr_key, settings_cr_value) + + request_cr_key = '/request' + request_cr_value = {k:bw_info[k] for k in MEC_CONSIDERED_FIELDS} + + config_rule = ConfigRule() + config_rule.action = ConfigActionEnum.CONFIGACTION_SET + config_rule_custom = ConfigRule_Custom() + config_rule_custom.resource_key = request_cr_key + config_rule_custom.resource_value = json.dumps(request_cr_value) + config_rule.custom.CopyFrom(config_rule_custom) + service_config_rules.append(config_rule) + + if 'sessionFilter' in bw_info: + a_ip = bw_info['sessionFilter']['sourceIp'] + z_ip = bw_info['sessionFilter']['dstAddress'] devices = client.ListDevices(Empty()).devices + router_id_counter = 1 for device in devices: + device_endpoint_uuids = {ep.name:ep.endpoint_id.endpoint_uuid.uuid for ep in device.device_endpoints} for cr in device.device_config.config_rules: - if cr.WhichOneof('config_rule') == 'custom' and cr.custom.resource_key == '_connect/settings': - for ep in json.loads(cr.custom.resource_value)['endpoints']: - if 'ip' in ep and (ep['ip'] == a_ip or ep['ip'] == z_ip): - ep_id = EndPointId() - ep_id.endpoint_uuid.uuid = ep['uuid'] - ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid - service.service_endpoint_ids.append(ep_id) - + if cr.WhichOneof('config_rule') != 'custom': + continue + match_subif = RE_CONFIG_RULE_IF_SUBIF.match(cr.custom.resource_key) + if not match_subif: + continue + address_ip = json.loads(cr.custom.resource_value).get('address_ip') + if address_ip not in [a_ip, z_ip]: + continue + port_name = 'PORT-' + match_subif.groups(0)[0] # `PORT-` added as prefix + ep_id = EndPointId() + ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] + ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid + service.service_endpoint_ids.append(ep_id) + + # add interface config rules + endpoint_settings_key = ENDPOINT_SETTINGS_KEY.format(device.name, port_name, VLAN_TAG) + if address_ip == a_ip: + field_updates = { + 'address_ip': (address_ip, True), + # 'router_id': ('.'.join([str(router_id_counter)]*4), True), + 'router_id': ('200.1.1.1', True), + 'neighbor_address_ip': ('192.168.150.2', True), + 'route_distinguisher': (route_distinguisher, True), + 'sub_interface_index': (0, True), + 'vlan_id' : (VLAN_TAG, True), + # 'bgp_as': (BGP_AS+router_id_counter, True), + 'bgp_as': (BGP_AS, True), + 'ip_address': (address_ip, True), + 'prefix_length': (PREFIX_LENGTH, True), + 'policy_AZ' : (policy_AZ, True), + 'policy_ZA' : (policy_ZA, True), + 'address_prefix' : (PREFIX_LENGTH, True), + } + elif address_ip == z_ip: + field_updates = { + 'address_ip': (address_ip, True), + # 'router_id': ('.'.join([str(router_id_counter)]*4), True), + 'router_id': ('200.1.1.2', True), + 'neighbor_address_ip': ('192.168.150.1', True), + 'route_distinguisher': (route_distinguisher, True), + 'sub_interface_index': (0, True), + 'vlan_id' : (VLAN_TAG, True), + # 'bgp_as': (BGP_AS+router_id_counter, True), + 'bgp_as': (BGP_AS, True), + 'ip_address': (address_ip, True), + 'prefix_length': (PREFIX_LENGTH, True), + 'policy_AZ' : (policy_ZA, True), + 'policy_ZA' : (policy_AZ, True), + 'address_prefix' : (PREFIX_LENGTH, True), + } + router_id_counter += 1 + LOGGER.debug(f'BEFORE UPDATE -> device.device_config.config_rules: {service_config_rules}') + update_config_rule_custom(service_config_rules, endpoint_settings_key, field_updates) + LOGGER.debug(f'AFTER UPDATE -> device.device_config.config_rules: {service_config_rules}') + + service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM - if 'appInsId' in bwInfo: - service.service_id.service_uuid.uuid = bwInfo['appInsId'] + if 'appInsId' in bw_info: + service.service_id.service_uuid.uuid = bw_info['appInsId'] service.service_id.context_id.context_uuid.uuid = 'admin' - service.name = bwInfo['appInsId'] + service.name = bw_info['appInsId'] - if 'fixedAllocation' in bwInfo: + if 'fixedAllocation' in bw_info: capacity = Constraint_SLA_Capacity() - capacity.capacity_gbps = float(bwInfo['fixedAllocation']) / 1.e9 + capacity.capacity_gbps = float(bw_info['fixedAllocation']) / 1.e9 constraint = Constraint() constraint.sla_capacity.CopyFrom(capacity) service.service_constraints.append(constraint) diff --git a/src/pathcomp/frontend/service/algorithms/tools/ComposeConfigRules.py b/src/pathcomp/frontend/service/algorithms/tools/ComposeConfigRules.py index 329552a91..498db7dcd 100644 --- a/src/pathcomp/frontend/service/algorithms/tools/ComposeConfigRules.py +++ b/src/pathcomp/frontend/service/algorithms/tools/ComposeConfigRules.py @@ -24,6 +24,7 @@ SETTINGS_RULE_NAME = '/settings' DEVICE_SETTINGS = re.compile(r'\/device\[([^\]]+)\]\/settings') ENDPOINT_SETTINGS = re.compile(r'\/device\[([^\]]+)\]\/endpoint\[([^\]]+)\]\/settings') +RE_ENDPOINT_VLAN_SETTINGS = re.compile(r'\/device\[([^\]]+)\]\/endpoint\[([^\]]+)\]\/vlan\[([^\]]+)\]\/settings') L2NM_SETTINGS_FIELD_DEFAULTS = { 'encapsulation_type': 'dot1q', @@ -150,6 +151,23 @@ def compose_device_config_rules( device_endpoint_keys = set(itertools.product(device_keys, endpoint_keys)) if len(device_endpoint_keys.intersection(endpoints_traversed)) == 0: continue subservice_config_rules.append(config_rule) + + match = RE_ENDPOINT_VLAN_SETTINGS.match(config_rule.custom.resource_key) + if match is not None: + device_uuid_or_name = match.group(1) + device_name_or_uuid = device_name_mapping[device_uuid_or_name] + device_keys = {device_uuid_or_name, device_name_or_uuid} + + endpoint_uuid_or_name = match.group(2) + endpoint_name_or_uuid_1 = endpoint_name_mapping[(device_uuid_or_name, endpoint_uuid_or_name)] + endpoint_name_or_uuid_2 = endpoint_name_mapping[(device_name_or_uuid, endpoint_uuid_or_name)] + endpoint_keys = {endpoint_uuid_or_name, endpoint_name_or_uuid_1, endpoint_name_or_uuid_2} + + device_endpoint_keys = set(itertools.product(device_keys, endpoint_keys)) + if len(device_endpoint_keys.intersection(endpoints_traversed)) == 0: continue + # ! check later: vlan removed from config_rule + config_rule.custom.resource_key = re.sub('\/vlan\[[^\]]+\]', '', config_rule.custom.resource_key) + subservice_config_rules.append(config_rule) else: continue diff --git a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py index 1e4425cdb..0369d6207 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py @@ -37,6 +37,7 @@ def setup_config_rules( vlan_id = json_endpoint_settings.get('vlan_id', 1 ) # 400 address_ip = json_endpoint_settings.get('address_ip', '0.0.0.0') # '2.2.2.1' address_prefix = json_endpoint_settings.get('address_prefix', 24 ) # 30 + neighbor_address_ip = json_endpoint_settings.get('neighbor_address_ip', '0.0.0.0') # '2.2.2.1' policy_import = json_endpoint_settings.get('policy_AZ', '2' ) # 2 policy_export = json_endpoint_settings.get('policy_ZA', '7' ) # 30 @@ -46,18 +47,21 @@ def setup_config_rules( network_subinterface_desc = json_endpoint_settings.get('subif_description','') #service_short_uuid = service_uuid.split('-')[-1] #network_instance_name = '{:s}-NetInst'.format(service_short_uuid) - network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 + # network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 + network_instance_name = 'default' - if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) + ''' + # if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) + if_subif_name = '{:s}'.format(endpoint_name[5:]) json_config_rules = [ - # Configure Interface (not used) - #json_config_rule_set( + # # Configure Interface (not used) + # json_config_rule_set( # '/interface[{:s}]'.format(endpoint_name), { # 'name': endpoint_name, # 'description': network_interface_desc, # 'mtu': mtu, - #}), + # }), #Create network instance json_config_rule_set( @@ -74,8 +78,10 @@ def setup_config_rules( json_config_rule_set( '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { 'name': network_instance_name, - 'protocol_name': 'BGP', + # 'protocol_name': 'BGP', + 'protocol_name': bgp_as, 'identifier': 'BGP', + # 'identifier': bgp_as, 'as': bgp_as, 'router_id': router_id, }), @@ -101,7 +107,8 @@ def setup_config_rules( json_config_rule_set( '/interface[{:s}]/subinterface[{:d}]'.format(if_subif_name, sub_interface_index), { 'name' : if_subif_name, - 'type' :'l3ipvlan', + # 'type' :'l3ipvlan', + 'type' :'ethernetCsmacd', 'mtu' : mtu, 'index' : sub_interface_index, 'description' : network_subinterface_desc, @@ -183,6 +190,40 @@ def setup_config_rules( }), ] + ''' + if_subif_name = '{:s}'.format(endpoint_name[5:]) + + json_config_rules = [ + + #Add DIRECTLY CONNECTED protocol to network instance + json_config_rule_set( + '/network_instance[{:s}]/protocols[DIRECTLY_CONNECTED]'.format(network_instance_name), { + 'name': network_instance_name, + 'identifier': 'DIRECTLY_CONNECTED', + 'protocol_name': 'DIRECTLY_CONNECTED', + }), + + # Add BGP neighbors + json_config_rule_set( + '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { + 'name': network_instance_name, + 'protocol_name': bgp_as, + 'identifier': 'BGP', + 'as': bgp_as, + 'router_id': router_id, + 'neighbors': [{'ip_address': neighbor_address_ip, 'remote_as': bgp_as}] + }), + json_config_rule_set( + '/network_instance[{:s}]/table_connections[DIRECTLY_CONNECTED][BGP][IPV4]'.format(network_instance_name), { + 'name' : network_instance_name, + 'src_protocol' : 'DIRECTLY_CONNECTED', + 'dst_protocol' : 'BGP', + 'address_family' : 'IPV4', + 'default_import_policy': 'ACCEPT_ROUTE', + }), + + ] + for res_key, res_value in endpoint_acls: json_config_rules.append( {'action': 1, 'acl': res_value} @@ -201,7 +242,8 @@ def teardown_config_rules( json_endpoint_settings : Dict = endpoint_settings.value service_short_uuid = service_uuid.split('-')[-1] - network_instance_name = '{:s}-NetInst'.format(service_short_uuid) + # network_instance_name = '{:s}-NetInst'.format(service_short_uuid) + network_instance_name = json_endpoint_settings.get('ni_name', service_short_uuid) #ELAN-AC:1 #network_interface_desc = '{:s}-NetIf'.format(service_uuid) #network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid) @@ -262,10 +304,10 @@ def teardown_config_rules( #Delete interface; automatically deletes: # - /interface[]/subinterface[] - json_config_rule_delete('/interface[{:s}]/subinterface[0]'.format(if_subif_name), - { - 'name': if_subif_name, - }), + # json_config_rule_delete('/interface[{:s}]/subinterface[0]'.format(if_subif_name), + # { + # 'name': if_subif_name, + # }), #Delete network instance; automatically deletes: # - /network_instance[]/interface[] -- GitLab From c4b613b68b5cc253fb652c3e47fa1f103cef96aa Mon Sep 17 00:00:00 2001 From: armingol Date: Wed, 14 Feb 2024 10:54:44 +0100 Subject: [PATCH 019/985] first version --- manifests/sliceservice.yaml | 2 +- src/common/tools/descriptor/Loader.py | 5 ++- src/common/tools/descriptor/Tools.py | 11 +++-- .../drivers/openconfig/templates/Inventory.py | 33 ++++++++++++++- .../drivers/openconfig/templates/Tools.py | 5 +++ src/slice/service/SliceServiceServicerImpl.py | 41 +++++++++++-------- .../service/slice_grouper/SliceGrouper.py | 28 ++++++------- src/webui/service/main/routes.py | 4 +- 8 files changed, 90 insertions(+), 39 deletions(-) diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index e7e5c1604..61f5b1d21 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: SLICE_GROUPING value: "DISABLE" envFrom: diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 916a73d30..9e536f935 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -104,7 +104,7 @@ class DescriptorLoader: self.__devices = self.__descriptors.get('devices' , []) self.__links = self.__descriptors.get('links' , []) self.__services = self.__descriptors.get('services' , []) - self.__slices = self.__descriptors.get('slices' , []) + self.__slices = self.__descriptors.get('data', []) #Coge de la file el campo slices self.__connections = self.__descriptors.get('connections', []) self.__contexts_add = None @@ -194,7 +194,7 @@ class DescriptorLoader: _slices = {} for slice_ in self.__slices: context_uuid = slice_['slice_id']['context_id']['context_uuid']['uuid'] - _slices.setdefault(context_uuid, []).append(slice_) + _slices.setdefault(context_uuid, []).append(slice_) #no tenemos context_uuid en este formato, lo meto a mano? return _slices @property @@ -215,6 +215,7 @@ class DescriptorLoader: # Format CustomConfigRules in Devices, Services and Slices provided in JSON format self.__devices = [format_device_custom_config_rules (device ) for device in self.__devices ] self.__services = [format_service_custom_config_rules(service) for service in self.__services] + LOGGERS.INFO(self.__slices) self.__slices = [format_slice_custom_config_rules (slice_ ) for slice_ in self.__slices ] # Context and Topology require to create the entity first, and add devices, links, services, diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index f03c635b8..1811f77d9 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -15,6 +15,7 @@ import copy, json from typing import Dict, List, Optional, Tuple, Union +#context es la db, al inicio esta vacía def get_descriptors_add_contexts(contexts : List[Dict]) -> List[Dict]: contexts_add = copy.deepcopy(contexts) for context in contexts_add: @@ -52,7 +53,7 @@ def get_descriptors_add_slices(slices : List[Dict]) -> List[Dict]: TypeResourceValue = Union[str, int, bool, float, dict, list] def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: for config_rule in config_rules: - if 'custom' not in config_rule: continue + # if 'custom' not in config_rule: continue #suponemos que siempre son custom, quitamos esta linea custom_resource_value : TypeResourceValue = config_rule['custom']['resource_value'] if isinstance(custom_resource_value, (dict, list)): custom_resource_value = json.dumps(custom_resource_value, sort_keys=True, indent=0) @@ -71,10 +72,14 @@ def format_service_custom_config_rules(service : Dict) -> Dict: service['service_config']['config_rules'] = config_rules return service +#UTILIZA LA FUNCION FORMAT_CUSTOM_CONFIG_RULES +#cambio def format_slice_custom_config_rules(slice_ : Dict) -> Dict: - config_rules = slice_.get('slice_config', {}).get('config_rules', []) + #donde cojo los config_rules + #las config_rules parecen estar en ACs? + config_rules = slice_.get('sdps', []) config_rules = format_custom_config_rules(config_rules) - slice_['slice_config']['config_rules'] = config_rules + slice_['sdps']['sdp']['attachment-circuits'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 2ae67ba47..01b417739 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -15,7 +15,7 @@ import logging, lxml.etree as ET from typing import Any, Dict, List, Tuple from .Namespace import NAMESPACES -from .Tools import add_value_from_tag +from .Tools import add_value_from_tag, add_int_from_tag LOGGER = logging.getLogger(__name__) @@ -56,6 +56,8 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] LOGGER.debug("InventoryPrueba") parent_types = {} + #Initialized count to 0 for index + count = 0 for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} @@ -64,6 +66,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: inventory['class'] = '' inventory['attributes'] = {} component_reference = [] + component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue @@ -84,6 +87,34 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + ##Added (after the checking of the name and the class) + #Physical index- Index of the component in the array + + add_int_from_tag(inventory['attributes'], 'physical-index', count) + count +=1 + + ##Added + #FRU + if inventory['class'] == 'FRU': + component_isfru = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) + add_value_from_tag(inventory['attributes'], 'isfru', component_isfru) + ##ID + component_id = xml_component.find('ocp:state/ocp:id', namespaces=NAMESPACES) + if not component_id is None: + add_value_from_tag(inventory['attributes'], 'id', component_id) + + ##OPER_STATUS + component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) + if not component_oper_status is None: + add_value_from_tag(inventory['attributes'], 'oper-status', component_oper_status) + + ##MODEL_ID + component_model_id = xml_component.find('ocp:state/ocp:entity-id', namespaces=NAMESPACES) + if not component_model_id is None: + add_value_from_tag(inventory['attributes'], 'model-id', component_model_id) + + ## + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 79bebef51..78e61e0ae 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -26,6 +26,11 @@ def add_value_from_tag(target : Dict, field_name: str, field_value : ET.Element, if cast is not None: field_value = cast(field_value) target[field_name] = field_value +def add_int_from_tag(target : Dict, field_name: str, field_value : int, cast=None) -> None: + if field_value is None: return + if cast is not None: field_value = cast(field_value) + target[field_name] = field_value + def add_value_from_collection(target : Dict, field_name: str, field_value : Collection) -> None: if field_value is None or len(field_value) == 0: return target[field_name] = field_value diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index cbe2dd5c7..52552a6ed 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +#agrupar slices agrupar recursos para no hacer mas configs + from typing import Optional import grpc, json, logging #, deepdiff from common.proto.context_pb2 import ( @@ -30,33 +32,34 @@ from interdomain.client.InterdomainClient import InterdomainClient from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper -LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) #crea un objeto de registro con el nombre del modulo actual METRICS_POOL = MetricsPool('Slice', 'RPC') -class SliceServiceServicerImpl(SliceServiceServicer): +class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio gRPC definido por SliceServiceServicer def __init__(self): LOGGER.debug('Creating Servicer...') - self._slice_grouper = SliceGrouper() + self._slice_grouper = SliceGrouper() #crea una instancia de slicegrouper LOGGER.debug('Servicer Created') def create_update(self, request : Slice) -> SliceId: # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is # being modified. context_client = ContextClient() - slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) + slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req - slice_rw = Slice() + slice_rw = Slice() #crea nueva slice desde la slice de la req slice_rw.CopyFrom(request if slice_ro is None else slice_ro) - if len(request.name) > 0: slice_rw.name = request.name + if len(request.name) > 0: slice_rw.name = request.name #actualizamos el nombre y estado de la slice rw slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member +#copiamos endpoints, reglas y configuraciones de la req a la slice copy_endpoint_ids(request.slice_endpoint_ids, slice_rw.slice_endpoint_ids ) # pylint: disable=no-member copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member - slice_id_with_uuids = context_client.SetSlice(slice_rw) + slice_id_with_uuids = context_client.SetSlice(slice_rw) #actualizar o crear la slice en la db if len(slice_rw.slice_endpoint_ids) < 2: # pylint: disable=no-member # unable to identify the kind of slice; just update endpoints, constraints and config rules @@ -65,8 +68,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): reply = context_client.SetSlice(slice_rw) context_client.close() return reply + #si tiene menos de 2 endpoints se omite la actualizacion y se retorna el sliceid - slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) #obtenemos la slice actualizada #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) @@ -74,8 +78,8 @@ class SliceServiceServicerImpl(SliceServiceServicer): #changes = deepdiff.DeepDiff(json_current_slice, json_updated_slice) #LOGGER.info('changes = {:s}'.format(str(changes))) - if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): - interdomain_client = InterdomainClient() + if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): #si la slice es interdominio + interdomain_client = InterdomainClient() #que es interdomain client? slice_id = interdomain_client.RequestSlice(slice_with_uuids) slice_ = context_client.GetSlice(slice_id) slice_active = Slice() @@ -97,10 +101,10 @@ class SliceServiceServicerImpl(SliceServiceServicer): service_client = ServiceClient() try: - _service = context_client.GetService(service_id) + _service = context_client.GetService(service_id) #obtener info de un servicio si existe except: # pylint: disable=bare-except # pylint: disable=no-member - service_request = Service() + service_request = Service() # sino se crea un nuevo servicio service_request.service_id.CopyFrom(service_id) service_request.service_type = ServiceTypeEnum.SERVICETYPE_UNKNOWN service_request.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED @@ -109,6 +113,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): service_request = Service() service_request.CopyFrom(_service) +#actualiza el servicio con la info de la slice # pylint: disable=no-member copy_endpoint_ids(request.slice_endpoint_ids, service_request.service_endpoint_ids) copy_constraints(request.slice_constraints, service_request.service_constraints) @@ -162,11 +167,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): slice_active.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_ACTIVE # pylint: disable=no-member context_client.SetSlice(slice_active) - service_client.close() - context_client.close() + service_client.close() #liberar recursos, que es realmente? + context_client.close() #db teraflow return slice_id - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) #agrega funcionalidades de metrica y seguridad def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: #try: # slice_ = context_client.GetSlice(request.slice_id) @@ -196,7 +201,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteSlice(self, request : SliceId, context : grpc.ServicerContext) -> Empty: - context_client = ContextClient() + context_client = ContextClient() #coge la info de una slice try: _slice = context_client.GetSlice(request) except: # pylint: disable=bare-except @@ -205,9 +210,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): _slice_rw = Slice() _slice_rw.CopyFrom(_slice) + #cambia el status _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) - +#elimina la slice considerando si es interdominio o no, y desagrupa la slice eliminada + #elimina los servicios asociados a la slice if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 2f1a79181..d59531a1b 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -14,7 +14,7 @@ import logging, pandas, threading from typing import Dict, Optional, Tuple -from sklearn.cluster import KMeans +from sklearn.cluster import KMeans #algoritmo de agrupamiento de scikit-learn (biblio de aprendizaje automatico) from common.proto.context_pb2 import Slice from common.tools.grpc.Tools import grpc_message_to_json_string from .Constants import SLICE_GROUPS @@ -27,30 +27,30 @@ LOGGER = logging.getLogger(__name__) class SliceGrouper: def __init__(self) -> None: - self._lock = threading.Lock() - self._is_enabled = is_slice_grouping_enabled() + self._lock = threading.Lock() #controla el acceso concurrente + self._is_enabled = is_slice_grouping_enabled() #esta habilitado el agrupamiento de slices? LOGGER.info('Slice Grouping: {:s}'.format('ENABLED' if self._is_enabled else 'DISABLED')) if not self._is_enabled: return - metrics_exporter = MetricsExporter() + metrics_exporter = MetricsExporter() #instancia de la clase metrics_exporter.create_table() - self._slice_groups = create_slice_groups(SLICE_GROUPS) + self._slice_groups = create_slice_groups(SLICE_GROUPS) #grupos de slices # Initialize and fit K-Means with the pre-defined clusters we want, i.e., one per slice group - df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) - k_means = KMeans(n_clusters=df_groups.shape[0]) + df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) #data frame con info de los grupos + k_means = KMeans(n_clusters=df_groups.shape[0]) #modelo utilizado para el agrupamiento k_means.fit(df_groups[['availability', 'capacity_gbps']]) df_groups['label'] = k_means.predict(df_groups[['availability', 'capacity_gbps']]) self._k_means = k_means self._df_groups = df_groups - self._group_mapping : Dict[str, Dict] = { + self._group_mapping : Dict[str, Dict] = { #Dict = dictionary group['name']:{k:v for k,v in group.items() if k != 'name'} - for group in list(df_groups.to_dict('records')) + for group in list(df_groups.to_dict('records')) #mapeo de nombres de grupo a sus atributos } - label_to_group = {} + label_to_group = {} #mapeo de etiquetas a nombres de grupo for group_name,group_attrs in self._group_mapping.items(): label = group_attrs['label'] availability = group_attrs['availability'] @@ -60,7 +60,7 @@ class SliceGrouper: label_to_group[label] = group_name self._label_to_group = label_to_group - def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: + def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: #selecciona un grupo para una slice with self._lock: grouping_parameters = get_slice_grouping_parameters(slice_obj) LOGGER.debug('[_select_group] grouping_parameters={:s}'.format(str(grouping_parameters))) @@ -78,16 +78,16 @@ class SliceGrouper: return group_name, availability, capacity_gbps @property - def is_enabled(self): return self._is_enabled + def is_enabled(self): return self._is_enabled #indica si el agrupamiento de slices esta habilitado - def group(self, slice_obj : Slice) -> bool: + def group(self, slice_obj : Slice) -> bool: #determina el grupo al que debe pertenecer la slice LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[group] selected_group={:s}'.format(str(selected_group))) if selected_group is None: return False return add_slice_to_group(slice_obj, selected_group) - def ungroup(self, slice_obj : Slice) -> bool: + def ungroup(self, slice_obj : Slice) -> bool: # desagrupa la slice de un grupo LOGGER.debug('[ungroup] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[ungroup] selected_group={:s}'.format(str(selected_group))) diff --git a/src/webui/service/main/routes.py b/src/webui/service/main/routes.py index 75f036bef..eba758ff3 100644 --- a/src/webui/service/main/routes.py +++ b/src/webui/service/main/routes.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64, json, logging #, re +import base64, json, logging +import traceback #, re from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request from common.proto.context_pb2 import ContextList, Empty, TopologyId, TopologyList from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications @@ -113,6 +114,7 @@ def home(): except Exception as e: # pylint: disable=broad-except LOGGER.exception('Descriptor load failed') flash(f'Descriptor load failed: `{str(e)}`', 'danger') + traceback.print_exc() # Agregar esta línea para imprimir el traceback completo finally: context_client.close() device_client.close() -- GitLab From 0b89701b198e73126e8419c571ea314eb21b26c5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 22 Feb 2024 12:16:07 +0000 Subject: [PATCH 020/985] Dataplane-in-a-box: - Updated Arista cEOS image version - Updated README.md --- dataplane-in-a-box/README.md | 1 + dataplane-in-a-box/arista.clab.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 45e5dc5e0..603b24114 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -46,6 +46,7 @@ docker exec -it clab-arista-wan1 bash ## Access cEOS CLI ```bash docker exec -it clab-arista-wan1 Cli +docker exec -it clab-arista-wan2 Cli ``` ## Configure ContainerLab clients diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 4f3b77129..3a92c7e9b 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -25,7 +25,7 @@ topology: arista_ceos: kind: arista_ceos #image: ceos:4.30.4M - image: ceos:4.31.1F + image: ceos:4.31.2F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest -- GitLab From 85ba5e458329734fe377eb7110151eccf9d4d966 Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 27 Feb 2024 08:23:26 +0000 Subject: [PATCH 021/985] bgp configuration with VRF support added --- .../templates/VPN/Interfaces_multivendor.py | 3 +- .../VPN/Network_instance_multivendor.py | 73 +++++++++++-------- .../templates/VPN/Routing_policy.py | 13 ++-- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 29 ++++---- .../l3nm_openconfig/ConfigRules.py | 65 ++++++----------- 5 files changed, 91 insertions(+), 92 deletions(-) diff --git a/src/device/service/drivers/openconfig/templates/VPN/Interfaces_multivendor.py b/src/device/service/drivers/openconfig/templates/VPN/Interfaces_multivendor.py index d6f72ee65..12411bd8a 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Interfaces_multivendor.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Interfaces_multivendor.py @@ -54,7 +54,8 @@ def create_If_SubIf(data,vendor, DEL): with tag('enabled'):text('true') with tag('subinterfaces'): with tag('subinterface'): - if vendor == 'ADVA': + # if vendor == 'ADVA': + if True: with tag('index'): text('0') with tag('config'): with tag('index'): text('0') diff --git a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py index e36955a0d..a8fc97cf9 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py @@ -64,10 +64,14 @@ def create_NI(parameters,vendor,DEL): elif "L3VRF" in parameters['type']: with tag('config'): with tag('name'):text(parameters['name']) - if vendor == "ADVA": + if "router_id" in parameters: + with tag('router-id'):text(parameters['router_id']) + # if vendor == "ADVA": + if True: with tag('type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"'):text('oc-ni-types:',parameters['type']) with tag('route-distinguisher'):text(parameters['route_distinguisher']) - if vendor == "ADVA": + # if vendor == "ADVA": + if True: with tag('encapsulation'): with tag('config'): with tag('encapsulation-type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"') :text('oc-ni-types:MPLS') @@ -116,9 +120,6 @@ def add_protocol_NI(parameters,vendor, DEL): else: with tag('network-instance'): with tag('name'):text(parameters['name']) - with tag('config'): - with tag('name'): text(parameters['name']) - with tag('type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"'): text('oc-ni-types:DEFAULT_INSTANCE') with tag('protocols'): with tag('protocol'): with tag('identifier', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) @@ -129,36 +130,25 @@ def add_protocol_NI(parameters,vendor, DEL): with tag('enabled'): text('true') if "BGP" in parameters['identifier']: with tag('bgp'): + with tag('name'): text(parameters['as']) with tag('global'): - with tag('afi-safis'): - with tag('afi-safi'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('config'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('enabled'): text('true') with tag('config'): with tag('as') :text(parameters['as']) - with tag('peer-groups'): - with tag('peer-group'): - with tag('peer-group-name'): text('IBGP') - with tag('config'): - with tag('peer-group-name'): text('IBGP') - with tag('peer-as'): text(parameters['protocol_name']) - with tag('afi-safis'): - with tag('afi-safi'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('config'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('enabled'): text('true') - + if "router_id" in parameters: + with tag('router-id'):text(parameters['router_id']) if 'neighbors' in parameters: with tag('neighbors'): - for neighbor in parameters['neighbors']: - with tag('neighbor'): - with tag('neighbor-address'): text(neighbor['ip_address']) - with tag('config'): - with tag('neighbor-address'): text(neighbor['ip_address']) - with tag('peer-group'): text('IBGP') + for neighbor in parameters['neighbors']: + with tag('neighbor'): + with tag('neighbor-address'): text(neighbor['ip_address']) + with tag('afi-safis'): + with tag('afi-safi', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): + with tag('afi-safi-name'): text('oc-bgp-types:IPV4_UNICAST') + with tag('enabled'): text('true') + with tag('config'): + with tag('neighbor-address'): text(neighbor['ip_address']) + with tag('enabled'): text('true') + with tag('peer-as'): text(parameters['as']) # if vendor == "ADVA": if True: with tag('tables'): @@ -207,6 +197,9 @@ def associate_If_to_NI(parameters, DEL): else: with tag('network-instance'): with tag('name'):text(parameters['name']) + with tag('config'): + with tag('name'):text(parameters['name']) + with tag('type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"'):text('oc-ni-types:L3VRF') with tag('interfaces'): with tag('interface'): with tag('id'):text(parameters['id']) @@ -214,6 +207,25 @@ def associate_If_to_NI(parameters, DEL): with tag('id') :text(parameters['id']) with tag('interface') :text(parameters['interface']) with tag('subinterface'):text(parameters['subinterface']) + with tag('interfaces', 'xmlns="http://openconfig.net/yang/interfaces"'): + with tag('interface'): + with tag('name'):text(parameters['interface']) + with tag('config'): + with tag('name'):text(parameters['interface']) + with tag('subinterfaces'): + with tag('subinterface'): + with tag('index'): text(parameters['subinterface']) + with tag('config'): + with tag('index'): text(parameters['subinterface']) + with tag('ipv4', 'xmlns="http://openconfig.net/yang/interfaces/ip"'): + with tag('config'): + with tag('mtu'): text('1500') + with tag('addresses'): + with tag('address'): + with tag('ip'):text(parameters['address_ip']) + with tag('config'): + with tag('ip'):text(parameters['address_ip']) + with tag('prefix-length'):text(parameters['address_prefix']) result = indent( doc.getvalue(), @@ -356,6 +368,7 @@ def create_table_conns(parameters,DEL): with tag('src-protocol','xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'): text('oc-pol-types:',parameters['src_protocol']) with tag('dst-protocol','xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'): text('oc-pol-types:',parameters['dst_protocol']) with tag('address-family', 'xmlns:oc-types="http://openconfig.net/yang/openconfig-types"'):text('oc-types:',parameters['address_family']) + with tag('dst-instance', 'xmlns="http://www.ipinfusion.com/yang/ocnos/ipi-oc-ni-augments"'):text('65000') if len(parameters['default_import_policy']) != 0: with tag('default-import-policy'):text(parameters['default_import_policy']) result = indent( diff --git a/src/device/service/drivers/openconfig/templates/VPN/Routing_policy.py b/src/device/service/drivers/openconfig/templates/VPN/Routing_policy.py index 54e6c1c01..e1287f9f8 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Routing_policy.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Routing_policy.py @@ -40,6 +40,7 @@ def create_rp_statement(data, DEL): RP_statement_name = data['statement_name'] RP_policy_result = data['policy_result'] RP_ext_comm_set_name = data['ext_community_set_name'] + RP_sequence_id = data['sequence_id'] with tag('routing-policy', xmlns="http://openconfig.net/yang/routing-policy"): @@ -55,9 +56,9 @@ def create_rp_statement(data, DEL): with tag('name'):text(RP_policy_name) with tag('statements'): with tag('statement'): - with tag('name'):text(RP_statement_name) + with tag('name'):text(RP_sequence_id) with tag('config'): - with tag('name'):text(RP_statement_name) + with tag('name'):text(RP_sequence_id) with tag('conditions'): with tag('config'): with tag('install-protocol-eq', **{'xmlns:openconfig-policy-types': 'http://openconfig.net/yang/policy-types'}):text('openconfig-policy-types:DIRECTLY_CONNECTED') @@ -133,14 +134,14 @@ data_2 = {'ext_community_member' : '65001:101', 'ext_community_set_name' : 'set_srv_101_a'} print('\nRouting Policy Statement - CREATE\n') -print(rp_statement(data_1, False)) +print(create_rp_statement(data_1, False)) print('\nRouting Policy Statement - DELETE\n') -print(rp_statement(data_1, True)) +print(create_rp_statement(data_1, True)) print('\nRouting Policy Defined Set - CREATE\n') -print(rp_defined_set(data_2, False)) +print(create_rp_def(data_2, False)) print('\nRouting Policy Defined Set - DELETE\n') -print(rp_defined_set(data_2, True)) +print(create_rp_def(data_2, True)) ''' ''' diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index 3c5d4b5e6..4b7ee7b39 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -38,8 +38,9 @@ ALLOCATION_DIRECTION_DESCRIPTIONS = { VLAN_TAG = 0 PREFIX_LENGTH = 24 BGP_AS = 65000 -policy_AZ = 'srv_{:d}_a'.format(VLAN_TAG) -policy_ZA = 'srv_{:d}_b'.format(VLAN_TAG) +POLICY_AZ = 'srv_{:d}_a'.format(VLAN_TAG) +POLICY_ZA = 'srv_{:d}_b'.format(VLAN_TAG) +BGP_NETWORK_CIDR = '192.168.150.0/24' def service_2_bwInfo(service: Service) -> dict: response = {} @@ -102,7 +103,6 @@ def bwInfo_2_service(client, bw_info: dict) -> Service: devices = client.ListDevices(Empty()).devices router_id_counter = 1 for device in devices: - device_endpoint_uuids = {ep.name:ep.endpoint_id.endpoint_uuid.uuid for ep in device.device_endpoints} for cr in device.device_config.config_rules: if cr.WhichOneof('config_rule') != 'custom': @@ -118,41 +118,42 @@ def bwInfo_2_service(client, bw_info: dict) -> Service: ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid service.service_endpoint_ids.append(ep_id) - # add interface config rules endpoint_settings_key = ENDPOINT_SETTINGS_KEY.format(device.name, port_name, VLAN_TAG) if address_ip == a_ip: field_updates = { 'address_ip': (address_ip, True), - # 'router_id': ('.'.join([str(router_id_counter)]*4), True), 'router_id': ('200.1.1.1', True), 'neighbor_address_ip': ('192.168.150.2', True), + 'self_bgp_interface_name': ('xe5', True), + 'self_bgp_interface_address_ip': ('192.168.150.1', True), + 'self_bgp_interface_address_prefix': (PREFIX_LENGTH, True), + 'self_bgp_sub_interface_index': (0, True), 'route_distinguisher': (route_distinguisher, True), 'sub_interface_index': (0, True), - 'vlan_id' : (VLAN_TAG, True), - # 'bgp_as': (BGP_AS+router_id_counter, True), 'bgp_as': (BGP_AS, True), 'ip_address': (address_ip, True), 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (policy_AZ, True), - 'policy_ZA' : (policy_ZA, True), + 'policy_AZ' : (POLICY_AZ, True), + 'policy_ZA' : (POLICY_ZA, True), 'address_prefix' : (PREFIX_LENGTH, True), } elif address_ip == z_ip: field_updates = { 'address_ip': (address_ip, True), - # 'router_id': ('.'.join([str(router_id_counter)]*4), True), 'router_id': ('200.1.1.2', True), 'neighbor_address_ip': ('192.168.150.1', True), + 'self_bgp_interface_name': ('xe5', True), + 'self_bgp_interface_address_ip': ('192.168.150.2', True), + 'self_bgp_interface_address_prefix': (PREFIX_LENGTH, True), + 'self_bgp_sub_interface_index': (0, True), 'route_distinguisher': (route_distinguisher, True), 'sub_interface_index': (0, True), - 'vlan_id' : (VLAN_TAG, True), - # 'bgp_as': (BGP_AS+router_id_counter, True), 'bgp_as': (BGP_AS, True), 'ip_address': (address_ip, True), 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (policy_ZA, True), - 'policy_ZA' : (policy_AZ, True), + 'policy_AZ' : (POLICY_ZA, True), + 'policy_ZA' : (POLICY_AZ, True), 'address_prefix' : (PREFIX_LENGTH, True), } router_id_counter += 1 diff --git a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py index 0369d6207..2f518d56a 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py @@ -47,11 +47,14 @@ def setup_config_rules( network_subinterface_desc = json_endpoint_settings.get('subif_description','') #service_short_uuid = service_uuid.split('-')[-1] #network_instance_name = '{:s}-NetInst'.format(service_short_uuid) - # network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 - network_instance_name = 'default' + network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 - ''' - # if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) + bgp_if_name = json_endpoint_settings.get('self_bgp_interface_name','') + bgp_address_ip = json_endpoint_settings.get('self_bgp_interface_address_ip','') + bgp_address_prefix = json_endpoint_settings.get('self_bgp_interface_address_prefix','') + bgp_sub_interface_index = json_endpoint_settings.get('self_bgp_sub_interface_index', 0 ) + + # if_subif_name = '{:s}.{:d}'.format(endpoint_name, 0) if_subif_name = '{:s}'.format(endpoint_name[5:]) json_config_rules = [ @@ -70,7 +73,7 @@ def setup_config_rules( 'description': network_interface_desc, 'type': 'L3VRF', 'route_distinguisher': route_distinguisher, - #'router_id': router_id, + 'router_id': router_id, #'address_families': address_families, }), @@ -78,12 +81,12 @@ def setup_config_rules( json_config_rule_set( '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { 'name': network_instance_name, - # 'protocol_name': 'BGP', 'protocol_name': bgp_as, 'identifier': 'BGP', - # 'identifier': bgp_as, + 'type': 'L3VRF', 'as': bgp_as, 'router_id': router_id, + 'neighbors': [{'ip_address': neighbor_address_ip, 'remote_as': bgp_as}] }), #Add DIRECTLY CONNECTED protocol to network instance @@ -107,7 +110,6 @@ def setup_config_rules( json_config_rule_set( '/interface[{:s}]/subinterface[{:d}]'.format(if_subif_name, sub_interface_index), { 'name' : if_subif_name, - # 'type' :'l3ipvlan', 'type' :'ethernetCsmacd', 'mtu' : mtu, 'index' : sub_interface_index, @@ -124,6 +126,18 @@ def setup_config_rules( 'id' : if_subif_name, 'interface' : if_subif_name, 'subinterface': sub_interface_index, + 'address_ip' : address_ip, + 'address_prefix': address_prefix, + }), + + json_config_rule_set( + '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, bgp_if_name), { + 'name' : network_instance_name, + 'id' : bgp_if_name, + 'interface' : bgp_if_name, + 'subinterface': bgp_sub_interface_index, + 'address_ip' : bgp_address_ip, + 'address_prefix': bgp_address_prefix, }), #Create routing policy @@ -139,6 +153,7 @@ def setup_config_rules( 'statement_name' : 'stm_{:s}'.format(policy_import), 'ext_community_set_name': 'set_{:s}'.format(policy_import), 'policy_result' : 'ACCEPT_ROUTE', + 'sequence_id' : '10' }), #Associate routing policy to network instance @@ -161,6 +176,7 @@ def setup_config_rules( 'statement_name' : 'stm_{:s}'.format(policy_export), 'ext_community_set_name': 'set_{:s}'.format(policy_export), 'policy_result' : 'ACCEPT_ROUTE', + 'sequence_id' : '10' }), #Associate routing policy to network instance @@ -189,39 +205,6 @@ def setup_config_rules( 'default_import_policy': 'ACCEPT_ROUTE', }), - ] - ''' - if_subif_name = '{:s}'.format(endpoint_name[5:]) - - json_config_rules = [ - - #Add DIRECTLY CONNECTED protocol to network instance - json_config_rule_set( - '/network_instance[{:s}]/protocols[DIRECTLY_CONNECTED]'.format(network_instance_name), { - 'name': network_instance_name, - 'identifier': 'DIRECTLY_CONNECTED', - 'protocol_name': 'DIRECTLY_CONNECTED', - }), - - # Add BGP neighbors - json_config_rule_set( - '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { - 'name': network_instance_name, - 'protocol_name': bgp_as, - 'identifier': 'BGP', - 'as': bgp_as, - 'router_id': router_id, - 'neighbors': [{'ip_address': neighbor_address_ip, 'remote_as': bgp_as}] - }), - json_config_rule_set( - '/network_instance[{:s}]/table_connections[DIRECTLY_CONNECTED][BGP][IPV4]'.format(network_instance_name), { - 'name' : network_instance_name, - 'src_protocol' : 'DIRECTLY_CONNECTED', - 'dst_protocol' : 'BGP', - 'address_family' : 'IPV4', - 'default_import_policy': 'ACCEPT_ROUTE', - }), - ] for res_key, res_value in endpoint_acls: -- GitLab From 8f65918fa23fbe19796917affff3f18bec4c62ff Mon Sep 17 00:00:00 2001 From: hajipour Date: Wed, 28 Feb 2024 12:29:12 +0000 Subject: [PATCH 022/985] service creation and deletion with webui added to pyangbind mode and tested --- .../drivers/openconfig/OpenConfigDriver.py | 3 + .../VPN/Network_instance_multivendor.py | 2 +- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 88 ++++++------ .../l3nm_openconfig/ConfigRules.py | 132 ++++++++++-------- .../L3NMOpenConfigServiceHandler.py | 6 +- 5 files changed, 131 insertions(+), 100 deletions(-) diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 99ae1c8db..452f2f7c8 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time import json import anytree, copy, logging, pytz, queue, re, threading #import lxml.etree as ET @@ -240,6 +241,8 @@ def edit_config( test_option=test_option, error_option=error_option, format=format) if commit_per_rule: netconf_handler.commit() # configuration commit + if 'table_connections' in resource_key: + time.sleep(5) # CPU usage might exceed critical level after route redistribution, BGP daemon needs time to reload #results[i] = True results.append(True) diff --git a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py index a8fc97cf9..13612b607 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py @@ -357,7 +357,7 @@ def create_table_conns(parameters,DEL): with tag('table-connection','xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" nc:operation="delete"'): with tag('src-protocol','xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'): text('oc-pol-types:',parameters['src_protocol']) with tag('dst-protocol','xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'): text('oc-pol-types:',parameters['dst_protocol']) - with tag('address-family', 'xmlns:oc-types="http://openconfig.net/yang/openconfig-types"'):text('oc-types:',parameters['dst_protocol']) + with tag('address-family', 'xmlns:oc-types="http://openconfig.net/yang/openconfig-types"'):text('oc-types:',parameters['address_family']) else: with tag('table-connections'): with tag('table-connection'): diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index 4b7ee7b39..f32fe5699 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -29,6 +29,7 @@ from common.tools.object_factory.Service import json_service_id LOGGER = logging.getLogger(__name__) ENDPOINT_SETTINGS_KEY = '/device[{:s}]/endpoint[{:s}]/vlan[{:d}]/settings' +DEVICE_SETTINGS_KEY = '/device[{:s}]/settings' RE_CONFIG_RULE_IF_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') MEC_CONSIDERED_FIELDS = ['requestType', 'sessionFilter', 'fixedAllocation', 'allocationDirection'] ALLOCATION_DIRECTION_DESCRIPTIONS = { @@ -40,7 +41,11 @@ PREFIX_LENGTH = 24 BGP_AS = 65000 POLICY_AZ = 'srv_{:d}_a'.format(VLAN_TAG) POLICY_ZA = 'srv_{:d}_b'.format(VLAN_TAG) -BGP_NETWORK_CIDR = '192.168.150.0/24' +BGP_NEIGHBOR_IP_A = '192.168.150.1' +BGP_NEIGHBOR_IP_Z = '192.168.150.2' +ROUTER_ID_A = '200.1.1.1' +ROUTER_ID_Z = '200.1.1.2' +ROUTE_DISTINGUISHER = '{:5d}:{:03d}'.format(BGP_AS, VLAN_TAG) def service_2_bwInfo(service: Service) -> dict: response = {} @@ -80,10 +85,6 @@ def bwInfo_2_service(client, bw_info: dict) -> Service: service_config_rules = service.service_config.config_rules - route_distinguisher = '{:5d}:{:03d}'.format(BGP_AS, VLAN_TAG) - settings_cr_key = '/settings' - settings_cr_value = {'bgp_as':(BGP_AS, True), 'route_distinguisher': (route_distinguisher, True)} - update_config_rule_custom(service_config_rules, settings_cr_key, settings_cr_value) request_cr_key = '/request' request_cr_value = {k:bw_info[k] for k in MEC_CONSIDERED_FIELDS} @@ -101,9 +102,10 @@ def bwInfo_2_service(client, bw_info: dict) -> Service: z_ip = bw_info['sessionFilter']['dstAddress'] devices = client.ListDevices(Empty()).devices - router_id_counter = 1 + ip_interface_name_dict = {} for device in devices: device_endpoint_uuids = {ep.name:ep.endpoint_id.endpoint_uuid.uuid for ep in device.device_endpoints} + skip_device = True for cr in device.device_config.config_rules: if cr.WhichOneof('config_rule') != 'custom': continue @@ -111,55 +113,59 @@ def bwInfo_2_service(client, bw_info: dict) -> Service: if not match_subif: continue address_ip = json.loads(cr.custom.resource_value).get('address_ip') + short_port_name = match_subif.groups(0)[0] + ip_interface_name_dict[address_ip] = short_port_name if address_ip not in [a_ip, z_ip]: continue - port_name = 'PORT-' + match_subif.groups(0)[0] # `PORT-` added as prefix + port_name = 'PORT-' + short_port_name # `PORT-` added as prefix ep_id = EndPointId() ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid service.service_endpoint_ids.append(ep_id) # add interface config rules endpoint_settings_key = ENDPOINT_SETTINGS_KEY.format(device.name, port_name, VLAN_TAG) - if address_ip == a_ip: - field_updates = { + if address_ip in a_ip: + router_id = ROUTER_ID_A + policy_az = POLICY_AZ + policy_za = POLICY_ZA + neighbor_bgp_interface_address_ip = BGP_NEIGHBOR_IP_Z + self_bgp_interface_address_ip = BGP_NEIGHBOR_IP_A + else: + router_id = ROUTER_ID_Z + policy_az = POLICY_ZA + policy_za = POLICY_AZ + neighbor_bgp_interface_address_ip= BGP_NEIGHBOR_IP_A + self_bgp_interface_address_ip = BGP_NEIGHBOR_IP_Z + endpoint_field_updates = { 'address_ip': (address_ip, True), - 'router_id': ('200.1.1.1', True), - 'neighbor_address_ip': ('192.168.150.2', True), - 'self_bgp_interface_name': ('xe5', True), - 'self_bgp_interface_address_ip': ('192.168.150.1', True), - 'self_bgp_interface_address_prefix': (PREFIX_LENGTH, True), - 'self_bgp_sub_interface_index': (0, True), - 'route_distinguisher': (route_distinguisher, True), - 'sub_interface_index': (0, True), - 'bgp_as': (BGP_AS, True), - 'ip_address': (address_ip, True), - 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (POLICY_AZ, True), - 'policy_ZA' : (POLICY_ZA, True), 'address_prefix' : (PREFIX_LENGTH, True), - } - elif address_ip == z_ip: - field_updates = { - 'address_ip': (address_ip, True), - 'router_id': ('200.1.1.2', True), - 'neighbor_address_ip': ('192.168.150.1', True), - 'self_bgp_interface_name': ('xe5', True), - 'self_bgp_interface_address_ip': ('192.168.150.2', True), - 'self_bgp_interface_address_prefix': (PREFIX_LENGTH, True), - 'self_bgp_sub_interface_index': (0, True), - 'route_distinguisher': (route_distinguisher, True), 'sub_interface_index': (0, True), - 'bgp_as': (BGP_AS, True), - 'ip_address': (address_ip, True), - 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (POLICY_ZA, True), - 'policy_ZA' : (POLICY_AZ, True), - 'address_prefix' : (PREFIX_LENGTH, True), } - router_id_counter += 1 LOGGER.debug(f'BEFORE UPDATE -> device.device_config.config_rules: {service_config_rules}') - update_config_rule_custom(service_config_rules, endpoint_settings_key, field_updates) + update_config_rule_custom(service_config_rules, endpoint_settings_key, endpoint_field_updates) LOGGER.debug(f'AFTER UPDATE -> device.device_config.config_rules: {service_config_rules}') + skip_device = False + if skip_device: + continue + device_field_updates = { + 'bgp_as':(BGP_AS, True), + 'route_distinguisher': (ROUTE_DISTINGUISHER, True), + 'router_id': (router_id, True), + 'policy_AZ': (policy_az, True), + 'policy_ZA': (policy_za, True), + 'neighbor_bgp_interface_address_ip': (neighbor_bgp_interface_address_ip, True), + 'self_bgp_interface_name': (ip_interface_name_dict[self_bgp_interface_address_ip], True), + 'self_bgp_interface_address_ip': (self_bgp_interface_address_ip, True), + 'bgp_interface_address_prefix': (PREFIX_LENGTH, True) + } + device_settings_key = DEVICE_SETTINGS_KEY.format(device.name) + LOGGER.debug(f'BEFORE UPDATE -> device.device_config.config_rules: {service_config_rules}') + update_config_rule_custom(service_config_rules, device_settings_key, device_field_updates) + LOGGER.debug(f'AFTER UPDATE -> device.device_config.config_rules: {service_config_rules}') + + settings_cr_key = '/settings' + settings_cr_value = {} + update_config_rule_custom(service_config_rules, settings_cr_key, settings_cr_value) service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM diff --git a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py index 2f518d56a..cee44641a 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py @@ -18,29 +18,30 @@ from service.service.service_handler_api.AnyTreeTools import TreeNode def setup_config_rules( service_uuid : str, connection_uuid : str, device_uuid : str, endpoint_uuid : str, endpoint_name : str, - service_settings : TreeNode, endpoint_settings : TreeNode, endpoint_acls : List [Tuple] + service_settings : TreeNode, endpoint_settings : TreeNode, endpoint_acls : List [Tuple], device_settings: TreeNode ) -> List[Dict]: if service_settings is None: return [] if endpoint_settings is None: return [] + if device_settings is None: return [] json_settings : Dict = service_settings.value json_endpoint_settings : Dict = endpoint_settings.value + json_device_settings : Dict = device_settings.value mtu = json_settings.get('mtu', 1450 ) # 1512 #address_families = json_settings.get('address_families', [] ) # ['IPV4'] - bgp_as = json_settings.get('bgp_as', 65000 ) # 65000 + bgp_as = json_device_settings.get('bgp_as', 65000 ) # 65000 - router_id = json_endpoint_settings.get('router_id', '0.0.0.0') # '10.95.0.10' - route_distinguisher = json_settings.get('route_distinguisher', '65000:101' ) # '60001:801' + router_id = json_device_settings.get('router_id', '0.0.0.0') # '10.95.0.10' + route_distinguisher = json_device_settings.get('route_distinguisher', '65000:101' ) # '60001:801' sub_interface_index = json_endpoint_settings.get('sub_interface_index', 0 ) # 1 vlan_id = json_endpoint_settings.get('vlan_id', 1 ) # 400 address_ip = json_endpoint_settings.get('address_ip', '0.0.0.0') # '2.2.2.1' address_prefix = json_endpoint_settings.get('address_prefix', 24 ) # 30 - neighbor_address_ip = json_endpoint_settings.get('neighbor_address_ip', '0.0.0.0') # '2.2.2.1' - policy_import = json_endpoint_settings.get('policy_AZ', '2' ) # 2 - policy_export = json_endpoint_settings.get('policy_ZA', '7' ) # 30 + policy_import = json_device_settings.get('policy_AZ', '2' ) # 2 + policy_export = json_device_settings.get('policy_ZA', '7' ) # 30 #network_interface_desc = '{:s}-NetIf'.format(service_uuid) network_interface_desc = json_endpoint_settings.get('ni_description','') #network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid) @@ -49,10 +50,11 @@ def setup_config_rules( #network_instance_name = '{:s}-NetInst'.format(service_short_uuid) network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 - bgp_if_name = json_endpoint_settings.get('self_bgp_interface_name','') - bgp_address_ip = json_endpoint_settings.get('self_bgp_interface_address_ip','') - bgp_address_prefix = json_endpoint_settings.get('self_bgp_interface_address_prefix','') - bgp_sub_interface_index = json_endpoint_settings.get('self_bgp_sub_interface_index', 0 ) + self_bgp_if_name = json_device_settings.get('self_bgp_interface_name', '') + self_bgp_address_ip = json_device_settings.get('self_bgp_interface_address_ip', '') + bgp_address_prefix = json_device_settings.get('bgp_interface_address_prefix', '') + bgp_sub_interface_index = json_device_settings.get('self_bgp_sub_interface_index', 0) + neighbor_bgp_if_address_ip= json_device_settings.get('neighbor_bgp_interface_address_ip', '0.0.0.0') # '2.2.2.1' # if_subif_name = '{:s}.{:d}'.format(endpoint_name, 0) if_subif_name = '{:s}'.format(endpoint_name[5:]) @@ -86,7 +88,7 @@ def setup_config_rules( 'type': 'L3VRF', 'as': bgp_as, 'router_id': router_id, - 'neighbors': [{'ip_address': neighbor_address_ip, 'remote_as': bgp_as}] + 'neighbors': [{'ip_address': neighbor_bgp_if_address_ip, 'remote_as': bgp_as}] }), #Add DIRECTLY CONNECTED protocol to network instance @@ -96,7 +98,6 @@ def setup_config_rules( 'identifier': 'DIRECTLY_CONNECTED', 'protocol_name': 'DIRECTLY_CONNECTED', }), - #Add STATIC protocol to network instance json_config_rule_set( @@ -131,12 +132,12 @@ def setup_config_rules( }), json_config_rule_set( - '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, bgp_if_name), { + '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, self_bgp_if_name), { 'name' : network_instance_name, - 'id' : bgp_if_name, - 'interface' : bgp_if_name, + 'id' : self_bgp_if_name, + 'interface' : self_bgp_if_name, 'subinterface': bgp_sub_interface_index, - 'address_ip' : bgp_address_ip, + 'address_ip' : self_bgp_address_ip, 'address_prefix': bgp_address_prefix, }), @@ -215,76 +216,75 @@ def setup_config_rules( def teardown_config_rules( service_uuid : str, connection_uuid : str, device_uuid : str, endpoint_uuid : str, endpoint_name : str, - service_settings : TreeNode, endpoint_settings : TreeNode + service_settings : TreeNode, endpoint_settings : TreeNode, device_settings: TreeNode ) -> List[Dict]: if service_settings is None: return [] if endpoint_settings is None: return [] + if device_settings is None: return [] json_settings : Dict = service_settings.value json_endpoint_settings : Dict = endpoint_settings.value + json_device_settings : Dict = device_settings.value service_short_uuid = service_uuid.split('-')[-1] # network_instance_name = '{:s}-NetInst'.format(service_short_uuid) network_instance_name = json_endpoint_settings.get('ni_name', service_short_uuid) #ELAN-AC:1 #network_interface_desc = '{:s}-NetIf'.format(service_uuid) - #network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid) + # network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid) + network_subinterface_desc = '' - #mtu = json_settings.get('mtu', 1450 ) # 1512 + mtu = json_settings.get('mtu', 1450 ) # 1512 #address_families = json_settings.get('address_families', [] ) # ['IPV4'] - #bgp_as = json_settings.get('bgp_as', 65000 ) # 65000 - route_distinguisher = json_settings.get('route_distinguisher', '0:0' ) # '60001:801' - #sub_interface_index = json_endpoint_settings.get('sub_interface_index', 0 ) # 1 - #router_id = json_endpoint_settings.get('router_id', '0.0.0.0') # '10.95.0.10' - vlan_id = json_endpoint_settings.get('vlan_id', 1 ) # 400 - #address_ip = json_endpoint_settings.get('address_ip', '0.0.0.0') # '2.2.2.1' - #address_prefix = json_endpoint_settings.get('address_prefix', 24 ) # 30 - policy_import = json_endpoint_settings.get('policy_AZ', '2' ) # 2 - policy_export = json_endpoint_settings.get('policy_ZA', '7' ) # 30 - - if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) + bgp_as = json_device_settings.get('bgp_as', 65000 ) # 65000 + route_distinguisher = json_device_settings.get('route_distinguisher', '0:0' ) # '60001:801' + sub_interface_index = json_endpoint_settings.get('sub_interface_index', 0 ) # 1 + router_id = json_device_settings.get('router_id', '0.0.0.0') # '10.95.0.10' + vlan_id = json_endpoint_settings.get('vlan_id', 1 ) # 400 + address_ip = json_endpoint_settings.get('address_ip', '0.0.0.0') # '2.2.2.1' + address_prefix = json_endpoint_settings.get('address_prefix', 24 ) # 30 + policy_import = json_device_settings.get('policy_AZ', '2' ) # 2 + policy_export = json_device_settings.get('policy_ZA', '7' ) # 30 - json_config_rules = [ - #Delete table connections - json_config_rule_delete( - '/network_instance[{:s}]/table_connections[DIRECTLY_CONNECTED][BGP][IPV4]'.format(network_instance_name),{ - 'name' : network_instance_name, - 'src_protocol' : 'DIRECTLY_CONNECTED', - 'dst_protocol' : 'BGP', - 'address_family': 'IPV4', - }), + self_bgp_if_name = json_device_settings.get('self_bgp_interface_name', '') + self_bgp_address_ip = json_device_settings.get('self_bgp_interface_address_ip', '') + bgp_address_prefix = json_device_settings.get('bgp_interface_address_prefix', '') + bgp_sub_interface_index = json_device_settings.get('self_bgp_sub_interface_index', 0) - - json_config_rule_delete( - '/network_instance[{:s}]/table_connections[STATIC][BGP][IPV4]'.format(network_instance_name), { - 'name' : network_instance_name, - 'src_protocol' : 'STATIC', - 'dst_protocol' : 'BGP', - 'address_family': 'IPV4', - }), + # if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) + if_subif_name = '{:s}'.format(endpoint_name[5:]) + json_config_rules = [ #Delete export routing policy - json_config_rule_delete( - '/routing_policy/policy_definition[{:s}_export]'.format(network_instance_name), { - 'policy_name': '{:s}_export'.format(network_instance_name), + # pylint: disable=duplicate-string-formatting-argument + '/routing_policy/policy_definition[{:s}_export]/statement[{:s}]'.format(policy_export, policy_export), { + 'policy_name' : policy_export, + 'statement_name' : 'stm_{:s}'.format(policy_export), + 'ext_community_set_name': 'set_{:s}'.format(policy_export), + 'policy_result' : 'ACCEPT_ROUTE', + 'sequence_id' : '10' }), json_config_rule_delete( '/routing_policy/bgp_defined_set[{:s}_rt_export][{:s}]'.format(policy_export, route_distinguisher), { 'ext_community_set_name': 'set_{:s}'.format(policy_export), + 'ext_community_member' : route_distinguisher, }), - #Delete import routing policy - json_config_rule_delete( - '/routing_policy/policy_definition[{:s}_import]'.format(network_instance_name), { - 'policy_name': '{:s}_import'.format(network_instance_name), + # pylint: disable=duplicate-string-formatting-argument + '/routing_policy/policy_definition[{:s}_import]/statement[{:s}]'.format(policy_import, policy_import), { + 'policy_name' : policy_import, + 'statement_name' : 'stm_{:s}'.format(policy_import), + 'ext_community_set_name': 'set_{:s}'.format(policy_import), + 'policy_result' : 'ACCEPT_ROUTE', + 'sequence_id' : '10' }), json_config_rule_delete( '/routing_policy/bgp_defined_set[{:s}_rt_import][{:s}]'.format(policy_import, route_distinguisher), { 'ext_community_set_name': 'set_{:s}'.format(policy_import), + 'ext_community_member' : route_distinguisher, }), - #Delete interface; automatically deletes: # - /interface[]/subinterface[] # json_config_rule_delete('/interface[{:s}]/subinterface[0]'.format(if_subif_name), @@ -296,6 +296,26 @@ def teardown_config_rules( # - /network_instance[]/interface[] # - /network_instance[]/protocols[] # - /network_instance[]/inter_instance_policies[] + + #Associate interface to network instance + json_config_rule_set( + '/network_instance[{:s}]/interface[{:s}]'.format('default', if_subif_name), { + 'name' : 'default', + 'id' : if_subif_name, + 'interface' : if_subif_name, + 'subinterface': sub_interface_index, + 'address_ip' : address_ip, + 'address_prefix': address_prefix, + }), + json_config_rule_set( + '/network_instance[{:s}]/interface[{:s}]'.format('default', self_bgp_if_name), { + 'name' : 'default', + 'id' : self_bgp_if_name, + 'interface' : self_bgp_if_name, + 'subinterface': bgp_sub_interface_index, + 'address_ip' : self_bgp_address_ip, + 'address_prefix': bgp_address_prefix, + }), json_config_rule_delete('/network_instance[{:s}]'.format(network_instance_name), { 'name': network_instance_name diff --git a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py index 3f8a6d9dd..d061573db 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py @@ -52,6 +52,7 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_settings = self.__settings_handler.get_device_settings(device_obj) endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) endpoint_acls = self.__settings_handler.get_endpoint_acls(device_obj, endpoint_obj) @@ -59,7 +60,7 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): json_config_rules = setup_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, endpoint_name, - settings, endpoint_settings, endpoint_acls) + settings, endpoint_settings, endpoint_acls, device_settings) if len(json_config_rules) > 0: del device_obj.device_config.config_rules[:] @@ -90,13 +91,14 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler): device_uuid, endpoint_uuid = get_device_endpoint_uuids(endpoint) device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid))) + device_settings = self.__settings_handler.get_device_settings(device_obj) endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid) endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj) endpoint_name = endpoint_obj.name json_config_rules = teardown_config_rules( service_uuid, connection_uuid, device_uuid, endpoint_uuid, endpoint_name, - settings, endpoint_settings) + settings, endpoint_settings, device_settings) if len(json_config_rules) > 0: del device_obj.device_config.config_rules[:] -- GitLab From f6afa0c7cebe50e35ad3abe59b1dbe15e06c8294 Mon Sep 17 00:00:00 2001 From: hajipour Date: Wed, 28 Feb 2024 16:28:34 +0000 Subject: [PATCH 023/985] service creation and deletion with webui added to jinja mode --- .../drivers/openconfig/OpenConfigDriver.py | 5 +-- .../drivers/openconfig/templates/__init__.py | 2 +- .../interface/subinterface/edit_config.xml | 13 ------- .../network_instance/edit_config.xml | 2 +- .../interface/edit_config.xml | 34 +++++++++++++++++++ .../protocols/edit_config.xml | 22 ++++++++++++ .../table_connections/edit_config.xml | 1 + .../statement/edit_config.xml | 4 +-- .../l3nm_openconfig/ConfigRules.py | 2 ++ 9 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 452f2f7c8..476c2f0cc 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -227,11 +227,8 @@ def edit_config( chk_length(str_resource_name, resource, min_length=2, max_length=2) resource_key,resource_value = resource chk_string(str_resource_name + '.key', resource_key, allow_empty=False) - str_config_messages = compose_config( # get template for configuration - resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer='pyangbind') - # str_config_messages = compose_config( # get template for configuration - # resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) + resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) for str_config_message in str_config_messages: # configuration of the received templates if str_config_message is None: raise UnsupportedResourceKeyException(resource_key) logger.debug('[{:s}] str_config_message[{:d}] = {:s}'.format( diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index 87eea1f0b..b1d8de511 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -121,7 +121,7 @@ def compose_config( # template generation templates.append(JINJA_ENV.get_template('acl/acl-set/acl-entry/edit_config.xml')) templates.append(JINJA_ENV.get_template('acl/interfaces/ingress/edit_config.xml')) data : Dict[str, Any] = json.loads(resource_value) - operation = 'delete' if delete else 'merge' + operation = 'delete' if delete else '' return [ '{:s}'.format( diff --git a/src/device/service/drivers/openconfig/templates/interface/subinterface/edit_config.xml b/src/device/service/drivers/openconfig/templates/interface/subinterface/edit_config.xml index e44100400..9cff1afd3 100644 --- a/src/device/service/drivers/openconfig/templates/interface/subinterface/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/interface/subinterface/edit_config.xml @@ -30,19 +30,6 @@ {% endif %} - {% if address_ip is defined %} - - - - {{address_ip}} - - {{address_ip}} - {{address_prefix}} - - - - - {% endif %} {% endif %} diff --git a/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml index 6b6b733da..891881867 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml @@ -1,5 +1,5 @@ - + {{name}} {% if operation is not defined or operation != 'delete' %} diff --git a/src/device/service/drivers/openconfig/templates/network_instance/interface/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/interface/edit_config.xml index 855f321b4..96e13cc37 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/interface/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/interface/edit_config.xml @@ -1,6 +1,10 @@ {{name}} + + {{name}} + oc-ni-types:L3VRF + {{id}} @@ -13,3 +17,33 @@ + + + {{interface}} + + {{interface}} + + + + {{subinterface}} + + {{subinterface}} + + + + 1500 + + +
+ {{address_ip}} + + {{address_ip}} + {{address_prefix}} + +
+
+
+
+
+
+
diff --git a/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml index c9c068e48..da66d97f0 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/protocols/edit_config.xml @@ -9,15 +9,37 @@ oc-pol-types:{{identifier}} {{protocol_name}} + true {% if identifier=='BGP' %} + {{as}} {{as}} {{router_id}} + {% if neighbors is defined %} + + {% for neighbor in neighbors %} + + {{neighbor['ip_address']}} + + + oc-bgp-types:IPV4_UNICAST + true + + + + {{neighbor['ip_address']}} + true + {{as}} + + + {% endfor %} + + {% endif %} {% endif %} {% endif %} diff --git a/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml b/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml index 46bf5e387..bd1dac1b3 100644 --- a/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml @@ -11,6 +11,7 @@ oc-pol-types:{{src_protocol}} oc-pol-types:{{dst_protocol}} oc-types:{{address_family}} + {{as}} {% if default_import_policy is defined %}{{default_import_policy}}{% endif %}
{% endif %} diff --git a/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml index eda2d99c9..1bf1cf322 100644 --- a/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml +++ b/src/device/service/drivers/openconfig/templates/routing_policy/policy_definition/statement/edit_config.xml @@ -8,9 +8,9 @@ - {{statement_name}} + {{sequence_id}} - {{statement_name}} + {{sequence_id}} diff --git a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py index cee44641a..87556c36b 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py @@ -195,6 +195,7 @@ def setup_config_rules( 'dst_protocol' : 'BGP', 'address_family' : 'IPV4', 'default_import_policy': 'ACCEPT_ROUTE', + 'as' : bgp_as, }), json_config_rule_set( @@ -204,6 +205,7 @@ def setup_config_rules( 'dst_protocol' : 'BGP', 'address_family' : 'IPV4', 'default_import_policy': 'ACCEPT_ROUTE', + 'as' : bgp_as, }), ] -- GitLab From 5f64f0ea74e6a69b63b177a2fd2adceca33f030a Mon Sep 17 00:00:00 2001 From: hajipour Date: Thu, 29 Feb 2024 21:38:18 +0000 Subject: [PATCH 024/985] service_2_bwInfo function bug resolved: change in config rules applied in the function --- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index f32fe5699..c39a380dc 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -31,7 +31,7 @@ LOGGER = logging.getLogger(__name__) ENDPOINT_SETTINGS_KEY = '/device[{:s}]/endpoint[{:s}]/vlan[{:d}]/settings' DEVICE_SETTINGS_KEY = '/device[{:s}]/settings' RE_CONFIG_RULE_IF_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') -MEC_CONSIDERED_FIELDS = ['requestType', 'sessionFilter', 'fixedAllocation', 'allocationDirection'] +MEC_CONSIDERED_FIELDS = ['requestType', 'sessionFilter', 'fixedAllocation', 'allocationDirection', 'fixedBWPriority'] ALLOCATION_DIRECTION_DESCRIPTIONS = { '00' : 'Downlink (towards the UE)', '01' : 'Uplink (towards the application/session)', @@ -59,12 +59,19 @@ def service_2_bwInfo(service: Service) -> dict: break for config_rule in service.service_config.config_rules: + resource_value_json = json.loads(config_rule.custom.resource_value) + if config_rule.custom.resource_key != '/request': + continue for key in ['allocationDirection', 'fixedBWPriority', 'requestType', 'sourceIp', 'sourcePort', 'dstPort', 'protocol', 'sessionFilter']: - if config_rule.custom.resource_key == key: - if key != 'sessionFilter': - response[key] = config_rule.custom.resource_value - else: - response[key] = json.loads(config_rule.custom.resource_value) + if key not in resource_value_json: + continue + + if key == 'sessionFilter': + response[key] = [resource_value_json[key]] + elif key == 'requestType': + response[key] = str(resource_value_json[key]) + else: + response[key] = resource_value_json[key] unixtime = time.time() response['timeStamp'] = { # Time stamp to indicate when the corresponding information elements are sent @@ -76,8 +83,6 @@ def service_2_bwInfo(service: Service) -> dict: def bwInfo_2_service(client, bw_info: dict) -> Service: # add description to allocationDirection code - if ad_code := bw_info.get('allocationDirection'): - bw_info['allocationDirection'] = {'code': ad_code, 'description': ALLOCATION_DIRECTION_DESCRIPTIONS[ad_code]} if 'sessionFilter' in bw_info: bw_info['sessionFilter'] = bw_info['sessionFilter'][0] # Discard other items in sessionFilter field -- GitLab From 5696254239b649a49c6155a618f7b54a928c786f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 1 Mar 2024 18:27:51 +0000 Subject: [PATCH 025/985] Dataplane-in-a-box: - Updated ContainerLab scenario - Updated TFS descriptors - Updated scripts - Updated README.md --- dataplane-in-a-box/README.md | 6 + dataplane-in-a-box/arista.clab.yml | 26 +++- dataplane-in-a-box/clab-cli-dc1.sh | 3 + dataplane-in-a-box/clab-cli-dc2.sh | 3 + .../{ceos-cli-wan1.sh => clab-cli-wan1.sh} | 0 .../{ceos-cli-wan2.sh => clab-cli-wan2.sh} | 0 dataplane-in-a-box/clab-cli-wan3.sh | 3 + dataplane-in-a-box/clab-load-image.sh | 19 --- dataplane-in-a-box/clab-pull-images.sh | 18 --- dataplane-in-a-box/links.json | 136 ------------------ .../{topology.json => tfs-01-topo-nodes.json} | 59 ++------ dataplane-in-a-box/tfs-02-topo-links.json | 63 ++++++++ dataplane-in-a-box/tfs-03-dc2dc-l2svc.json | 17 +++ ...3-service.json => tfs-04-dc2dc-l3svc.json} | 18 +-- 14 files changed, 139 insertions(+), 232 deletions(-) create mode 100755 dataplane-in-a-box/clab-cli-dc1.sh create mode 100755 dataplane-in-a-box/clab-cli-dc2.sh rename dataplane-in-a-box/{ceos-cli-wan1.sh => clab-cli-wan1.sh} (100%) rename dataplane-in-a-box/{ceos-cli-wan2.sh => clab-cli-wan2.sh} (100%) create mode 100755 dataplane-in-a-box/clab-cli-wan3.sh delete mode 100755 dataplane-in-a-box/clab-load-image.sh delete mode 100755 dataplane-in-a-box/clab-pull-images.sh delete mode 100644 dataplane-in-a-box/links.json rename dataplane-in-a-box/{topology.json => tfs-01-topo-nodes.json} (50%) create mode 100644 dataplane-in-a-box/tfs-02-topo-links.json create mode 100644 dataplane-in-a-box/tfs-03-dc2dc-l2svc.json rename dataplane-in-a-box/{dc-2-dc-l3-service.json => tfs-04-dc2dc-l3svc.json} (72%) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 603b24114..4dd22dec3 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -19,6 +19,12 @@ source dataplane-in-a-box/deploy_specs.sh sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 ``` +## Download Arista cEOS image and create Docker image +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +docker import arista/cEOS64-lab-4.31.2F.tar ceos:4.31.2F +``` + ## Deploy scenario ```bash cd ~/tfs-ctrl/dataplane-in-a-box diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 3a92c7e9b..2865100bd 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -35,21 +35,37 @@ topology: kind: arista_ceos mgmt-ipv4: 172.20.20.101 ports: [6001:6030] + wan2: kind: arista_ceos mgmt-ipv4: 172.20.20.102 ports: [6002:6030] - client1: + wan3: + kind: arista_ceos + mgmt-ipv4: 172.20.20.103 + ports: [6003:6030] + + dc1: kind: linux mgmt-ipv4: 172.20.20.201 ports: [2201:22] - client2: + exec: + - ip link set address 00:c1:ab:00:00:01 dev eth1 + - ip address add 192.168.1.10/24 dev eth1 + - ip route add 192.168.2.0/24 via 192.168.1.1 + + dc2: kind: linux mgmt-ipv4: 172.20.20.202 ports: [2202:22] + exec: + - ip link set address 00:c1:ab:00:00:02 dev eth1 + - ip address add 192.168.2.10/24 dev eth1 + - ip route add 192.168.1.0/24 via 192.168.2.1 links: - - endpoints: ["wan1:eth1", "wan2:eth1"] - - endpoints: ["client1:eth1", "wan1:eth10"] - - endpoints: ["client2:eth1", "wan2:eth10"] + - endpoints: ["wan1:eth2", "wan2:eth1"] + - endpoints: ["wan2:eth3", "wan3:eth2"] + - endpoints: ["wan1:eth10", "dc1:eth1"] + - endpoints: ["wan3:eth10", "dc2:eth1"] diff --git a/dataplane-in-a-box/clab-cli-dc1.sh b/dataplane-in-a-box/clab-cli-dc1.sh new file mode 100755 index 000000000..7d793f035 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc1 bash diff --git a/dataplane-in-a-box/clab-cli-dc2.sh b/dataplane-in-a-box/clab-cli-dc2.sh new file mode 100755 index 000000000..311d6dae5 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc2 bash diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/clab-cli-wan1.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan1.sh rename to dataplane-in-a-box/clab-cli-wan1.sh diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/clab-cli-wan2.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan2.sh rename to dataplane-in-a-box/clab-cli-wan2.sh diff --git a/dataplane-in-a-box/clab-cli-wan3.sh b/dataplane-in-a-box/clab-cli-wan3.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-wan3.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh deleted file mode 100755 index 87e666422..000000000 --- a/dataplane-in-a-box/clab-load-image.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download image from Arista account > software downloads - -cd /home/tfs/tfs-ctrl/dataplane-in-a-box -docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh deleted file mode 100755 index 8f2805c6b..000000000 --- a/dataplane-in-a-box/clab-pull-images.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker pull ghcr.io/hellt/network-multitool:latest -#docker pull ghcr.io/nokia/srlinux:23.7.2 -#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json deleted file mode 100644 index 832a24fdd..000000000 --- a/dataplane-in-a-box/links.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "contexts": [ - {"context_id": {"context_uuid": {"uuid": "admin"}}} - ], - "topologies": [ - {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} - ], - "devices": [ - { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - } - ], - "links": [ - { - "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - } - ] -} diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/tfs-01-topo-nodes.json similarity index 50% rename from dataplane-in-a-box/topology.json rename to dataplane-in-a-box/tfs-01-topo-nodes.json index 42752235d..3b5e42b99 100644 --- a/dataplane-in-a-box/topology.json +++ b/dataplane-in-a-box/tfs-01-topo-nodes.json @@ -7,85 +7,54 @@ ], "devices": [ { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc1"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc2"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "wan1"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": false - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan2"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan3"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.103"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} } - ], - "links": [] + ] } diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json new file mode 100644 index 000000000..b9070dd9d --- /dev/null +++ b/dataplane-in-a-box/tfs-02-topo-links.json @@ -0,0 +1,63 @@ +{ + "links": [ + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/1==wan1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/3==wan3/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/2==wan2/ethernet-1/3"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc1/eth1==wan1/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/10==dc1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan3/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/10==dc2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json new file mode 100644 index 000000000..8d10e5f4b --- /dev/null +++ b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l2svc"} + }, + "service_type": 2, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": []} + } + ] +} diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json similarity index 72% rename from dataplane-in-a-box/dc-2-dc-l3-service.json rename to dataplane-in-a-box/tfs-04-dc2dc-l3svc.json index cb9ef972e..b21cba0da 100644 --- a/dataplane-in-a-box/dc-2-dc-l3-service.json +++ b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json @@ -2,33 +2,33 @@ "services": [ { "service_id": { - "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l3svc"} }, "service_type": 1, "service_status": {"service_status": 1}, "service_endpoint_ids": [ - {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, - {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} ], "service_constraints": [], "service_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}} ]} -- GitLab From ee3d0767356b3b7c198fd28d4737bc97a8eaa72d Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 11 Mar 2024 08:43:30 +0000 Subject: [PATCH 026/985] Changes are made to activate monitoring module, e2e Orchestrator, enable drop cocDB --- my_deploy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index 0fcb51f90..5bd58d0dc 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -23,7 +23,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" # Uncomment to activate Monitoring -#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate ZTP #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" @@ -44,7 +44,7 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene #export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" # Uncomment to activate E2E Orchestrator -#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" @@ -93,7 +93,7 @@ export CRDB_DATABASE="tfs" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="" +export CRDB_DROP_DATABASE_IF_EXISTS="YES" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" @@ -141,7 +141,7 @@ export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" # Disable flag for dropping tables if they exist. -export QDB_DROP_TABLES_IF_EXIST="" +export QDB_DROP_TABLES_IF_EXIST="YES" # Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" -- GitLab From afc009ba8234cc7d6d2f291d92a0f35679e2e492 Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 12 Mar 2024 11:45:33 +0000 Subject: [PATCH 027/985] feat: e2e acl installation added: 1.post, get, delete endpoints of acl added to nbi 2.ipinfusion netconf proprietary acl recipe added since openconfig recipe does not create acl entries in the router --- proto/acl.proto | 1 + proto/context.proto | 1 + .../drivers/openconfig/templates/__init__.py | 29 +++- .../openconfig/templates/acl/__init__.py | 13 ++ .../edit_config_ipinfusion_proprietary.xml | 34 ++++ .../openconfig/templates/acl/acl_adapter.py | 75 ++++++++ .../acl/acl_adapter_ipinfusion_proprietary.py | 65 +++++++ .../edit_config_ipinfusion_proprietary.xml | 26 +++ src/nbi/requirements.in | 1 + src/nbi/service/__main__.py | 2 + .../nbi_plugins/ietf_acl/__init__.py | 42 +++++ .../nbi_plugins/ietf_acl/acl_service.py | 98 +++++++++++ .../nbi_plugins/ietf_acl/acl_services.py | 68 ++++++++ .../nbi_plugins/ietf_acl/ietf_acl_parser.py | 164 ++++++++++++++++++ src/nbi/tests/data/ietf_acl.json | 56 ++++++ 15 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 src/device/service/drivers/openconfig/templates/acl/__init__.py create mode 100644 src/device/service/drivers/openconfig/templates/acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml create mode 100644 src/device/service/drivers/openconfig/templates/acl/acl_adapter.py create mode 100644 src/device/service/drivers/openconfig/templates/acl/acl_adapter_ipinfusion_proprietary.py create mode 100644 src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml create mode 100644 src/nbi/service/rest_server/nbi_plugins/ietf_acl/__init__.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_service.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_services.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_parser.py create mode 100644 src/nbi/tests/data/ietf_acl.json diff --git a/proto/acl.proto b/proto/acl.proto index 3dba735dc..f691c3dbd 100644 --- a/proto/acl.proto +++ b/proto/acl.proto @@ -46,6 +46,7 @@ message AclMatch { uint32 dst_port = 6; uint32 start_mpls_label = 7; uint32 end_mpls_label = 8; + string flags = 9; } message AclAction { diff --git a/proto/context.proto b/proto/context.proto index d5022ac29..1f8b0ce19 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -498,6 +498,7 @@ message ConfigRule_Custom { message ConfigRule_ACL { EndPointId endpoint_id = 1; acl.AclRuleSet rule_set = 2; + string interface = 3; } message ConfigRule { diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index 87eea1f0b..dcd3cf683 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -27,6 +27,9 @@ from .NetworkInstances import parse as parse_network_instances from .RoutingPolicy import parse as parse_routing_policy from .Acl import parse as parse_acl from .Inventory import parse as parse_inventory +from .acl.acl_adapter import acl_cr_to_dict +from .acl.acl_adapter_ipinfusion_proprietary import acl_cr_to_dict_ipinfusion_proprietary + LOGGER = logging.getLogger(__name__) ALL_RESOURCE_KEYS = [ @@ -113,16 +116,26 @@ def compose_config( # template generation elif (message_renderer == "jinja"): templates =[] - template_name = '{:s}/edit_config.xml'.format(RE_REMOVE_FILTERS.sub('', resource_key)) - templates.append(JINJA_ENV.get_template(template_name)) - if "acl_ruleset" in resource_key: # MANAGING ACLs - templates =[] - templates.append(JINJA_ENV.get_template('acl/acl-set/acl-entry/edit_config.xml')) - templates.append(JINJA_ENV.get_template('acl/interfaces/ingress/edit_config.xml')) - data : Dict[str, Any] = json.loads(resource_value) + if True: #vendor == 'ipinfusion': #! ipinfusion proprietary netconf receipe is used temporarily + acl_entry_path = 'acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml' + acl_ingress_path = 'acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml' + data : Dict[str, Any] = acl_cr_to_dict_ipinfusion_proprietary(resource_value, delete=delete) + else: + acl_entry_path = 'acl/acl-set/acl-entry/edit_config.xml' + acl_ingress_path = 'acl/interfaces/ingress/edit_config.xml' + data : Dict[str, Any] = acl_cr_to_dict(resource_value, delete=delete) + if delete: # unpair acl and interface before removing acl + templates.append(JINJA_ENV.get_template(acl_ingress_path)) + templates.append(JINJA_ENV.get_template(acl_entry_path)) + else: + templates.append(JINJA_ENV.get_template(acl_entry_path)) + templates.append(JINJA_ENV.get_template(acl_ingress_path)) + else: + template_name = '{:s}/edit_config.xml'.format(RE_REMOVE_FILTERS.sub('', resource_key)) + templates.append(JINJA_ENV.get_template(template_name)) + data : Dict[str, Any] = json.loads(resource_value) operation = 'delete' if delete else 'merge' - return [ '{:s}'.format( template.render(**data, operation=operation, vendor=vendor).strip()) diff --git a/src/device/service/drivers/openconfig/templates/acl/__init__.py b/src/device/service/drivers/openconfig/templates/acl/__init__.py new file mode 100644 index 000000000..f80ccfd52 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/src/device/service/drivers/openconfig/templates/acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml b/src/device/service/drivers/openconfig/templates/acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml new file mode 100644 index 000000000..d0210a66c --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml @@ -0,0 +1,34 @@ + + + + {{name}} + {% if type is defined %}{{type}}{% endif %} + + {{name}} + {% if type is defined %}{{type}}{% endif %} + + {% if operation != 'delete' %} + + + {{sequence_id}} + + {{sequence_id}} + + + + {{source_address}} + {{destination_address}} + {{dscp}} + + {{source_port}} + {{destination_port}} + {{tcp_flags}} + {{forwarding_action}} + + + + + {% endif %} + + + \ No newline at end of file diff --git a/src/device/service/drivers/openconfig/templates/acl/acl_adapter.py b/src/device/service/drivers/openconfig/templates/acl/acl_adapter.py new file mode 100644 index 000000000..244c4b616 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/acl_adapter.py @@ -0,0 +1,75 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, TypedDict + +from ..ACL.ACL_multivendor import RULE_TYPE_MAPPING, FORWARDING_ACTION_MAPPING, LOG_ACTION_MAPPING + +class ACLRequestData(TypedDict): + name: str # acl-set name + type: str # acl-set type + sequence_id: int # acl-entry sequence-id + source_address: str + destination_address: str + forwarding_action: str + id: str # interface id + interface: str + subinterface: int + set_name_ingress: str # ingress-acl-set name + type_ingress: str # ingress-acl-set type + all: bool + dscp: int + protocol: int + tcp_flags: str + source_port: int + destination_port: int + +def acl_cr_to_dict(acl_cr_dict: Dict, subinterface:int = 0) -> Dict: + rule_set = acl_cr_dict['rule_set'] + rule_set_entry = rule_set['entries'][0] + rule_set_entry_match = rule_set_entry['match'] + rule_set_entry_action = rule_set_entry['action'] + + name: str = rule_set['name'] + type: str = RULE_TYPE_MAPPING[rule_set["type"]] + sequence_id = rule_set_entry['sequence_id'] + source_address = rule_set_entry_match['src_address'] + destination_address = rule_set_entry_match['dst_address'] + forwarding_action: str = FORWARDING_ACTION_MAPPING[rule_set_entry_action['forward_action']] + interface_id = acl_cr_dict['interface'] + interface = interface_id + set_name_ingress = name + type_ingress = type + + return ACLRequestData( + name=name, + type=type, + sequence_id=sequence_id, + source_address=source_address, + destination_address=destination_address, + forwarding_action=forwarding_action, + id=interface_id, + interface=interface, + # subinterface=subinterface, + set_name_ingress=set_name_ingress, + type_ingress=type_ingress, + all=True, + dscp=18, + protocol=6, + tcp_flags='TCP_SYN', + source_port=22, + destination_port=80 + ) + + \ No newline at end of file diff --git a/src/device/service/drivers/openconfig/templates/acl/acl_adapter_ipinfusion_proprietary.py b/src/device/service/drivers/openconfig/templates/acl/acl_adapter_ipinfusion_proprietary.py new file mode 100644 index 000000000..79db6ad98 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/acl_adapter_ipinfusion_proprietary.py @@ -0,0 +1,65 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, TypedDict + + +RULE_TYPE_MAPPING = { + 'ACLRULETYPE_IPV4' : 'ip', +} + +FORWARDING_ACTION_MAPPING = { + 'ACLFORWARDINGACTION_DROP' : 'deny', + 'ACLFORWARDINGACTION_ACCEPT' : 'permit', +} + +class ACLRequestData(TypedDict): + name: str # acl-set name + type: str # acl-set type + sequence_id: int # acl-entry sequence-id + source_address: str + destination_address: str + forwarding_action: str + interface: str + dscp: int + tcp_flags: str + source_port: int + destination_port: int + +def acl_cr_to_dict_ipinfusion_proprietary(acl_cr_dict: Dict, delete: bool = False) -> Dict: + rule_set = acl_cr_dict['rule_set'] + name: str = rule_set['name'] + type: str = RULE_TYPE_MAPPING[rule_set["type"]] + interface = acl_cr_dict['interface'][5:] # remove preceding `PORT-` characters + if delete: + return ACLRequestData(name=name, type=type, interface=interface) + rule_set_entry = rule_set['entries'][0] + rule_set_entry_match = rule_set_entry['match'] + rule_set_entry_action = rule_set_entry['action'] + + return ACLRequestData( + name=name, + type=type, + sequence_id=rule_set_entry['sequence_id'], + source_address=rule_set_entry_match['src_address'], + destination_address=rule_set_entry_match['dst_address'], + forwarding_action=FORWARDING_ACTION_MAPPING[rule_set_entry_action['forward_action']], + interface=interface, + dscp=rule_set_entry_match["dscp"], + tcp_flags=rule_set_entry_match["flags"], + source_port=rule_set_entry_match['src_port'], + destination_port=rule_set_entry_match['dst_port'] + ) + + \ No newline at end of file diff --git a/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml b/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml new file mode 100644 index 000000000..6e502154f --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml @@ -0,0 +1,26 @@ + + + + {{interface}} + + {{interface}} + + + + {% if type is defined %}{{type}}{% endif %} + + + {{name}} + + {{name}} + + + + + {% if type is defined %}{{type}}{% endif %} + + + + + + \ No newline at end of file diff --git a/src/nbi/requirements.in b/src/nbi/requirements.in index 6e3eb9440..7a7b1cffb 100644 --- a/src/nbi/requirements.in +++ b/src/nbi/requirements.in @@ -24,3 +24,4 @@ pyang==2.6.0 git+https://github.com/robshakir/pyangbind.git requests==2.27.1 werkzeug==2.3.7 +pydantic==2.6.3 diff --git a/src/nbi/service/__main__.py b/src/nbi/service/__main__.py index 8834e45a2..2a8a2251d 100644 --- a/src/nbi/service/__main__.py +++ b/src/nbi/service/__main__.py @@ -26,6 +26,7 @@ from .rest_server.nbi_plugins.ietf_l2vpn import register_ietf_l2vpn from .rest_server.nbi_plugins.ietf_l3vpn import register_ietf_l3vpn from .rest_server.nbi_plugins.ietf_network import register_ietf_network from .rest_server.nbi_plugins.ietf_network_slice import register_ietf_nss +from .rest_server.nbi_plugins.ietf_acl import register_ietf_acl terminate = threading.Event() LOGGER = None @@ -68,6 +69,7 @@ def main(): register_ietf_l3vpn(rest_server) # Registering L3VPN entrypoint register_ietf_network(rest_server) register_ietf_nss(rest_server) # Registering NSS entrypoint + register_ietf_acl(rest_server) rest_server.start() # Wait for Ctrl+C or termination signal diff --git a/src/nbi/service/rest_server/nbi_plugins/ietf_acl/__init__.py b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/__init__.py new file mode 100644 index 000000000..6c1353bff --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask_restful import Resource + +from nbi.service.rest_server.nbi_plugins.ietf_acl.acl_service import ACL +from nbi.service.rest_server.nbi_plugins.ietf_acl.acl_services import ACLs +from nbi.service.rest_server.RestServer import RestServer + +URL_PREFIX = "/restconf/data" + + +def __add_resource(rest_server: RestServer, resource: Resource, *urls, **kwargs): + urls = [(URL_PREFIX + url) for url in urls] + rest_server.add_resource(resource, *urls, **kwargs) + + +def register_ietf_acl(rest_server: RestServer): + __add_resource( + rest_server, + ACLs, + "/device=/ietf-access-control-list:acls", + "/device=/ietf-access-control-list:acls", + ) + + __add_resource( + rest_server, + ACL, + "/device=/ietf-access-control-list:acl=", + "/device=/ietf-access-control-list:acl=/", + ) diff --git a/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_service.py b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_service.py new file mode 100644 index 000000000..466a68efc --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_service.py @@ -0,0 +1,98 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import json + +from flask_restful import Resource +from werkzeug.exceptions import NotFound + +from nbi.service.rest_server.nbi_plugins.tools.Authentication import HTTP_AUTH +from common.proto.acl_pb2 import AclRuleTypeEnum +from common.proto.context_pb2 import ( + ConfigActionEnum, + ConfigRule, + Device, + DeviceId, +) +from common.tools.object_factory.Device import json_device_id +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient + + +from .ietf_acl_parser import ietf_acl_from_config_rule_resource_value + +LOGGER = logging.getLogger(__name__) + +ACL_CONIG_RULE_KEY = r'\/device\[.+\]\/endpoint\[(.+)\]/acl_ruleset\[{}\]' + + +class ACL(Resource): + # @HTTP_AUTH.login_required + def get(self, device_uuid: str, acl_name: str): + RE_ACL_CONIG_RULE_KEY = re.compile(ACL_CONIG_RULE_KEY.format(acl_name)) + + context_client = ContextClient() + device_client = DeviceClient() + + _device = context_client.GetDevice(DeviceId(**json_device_id(device_uuid))) + + + for cr in _device.device_config.config_rules: + if cr.WhichOneof('config_rule') == 'custom': + if ep_uuid_match := RE_ACL_CONIG_RULE_KEY.match(cr.custom.resource_key): + endpoint_uuid = ep_uuid_match.groups(0)[0] + resource_value_dict = json.loads(cr.custom.resource_value) + LOGGER.debug(f'P99: {resource_value_dict}') + return ietf_acl_from_config_rule_resource_value(resource_value_dict) + else: + raise NotFound(f'ACL not found') + + # @HTTP_AUTH.login_required + def delete(self, device_uuid: str, acl_name: str): + RE_ACL_CONIG_RULE_KEY = re.compile(ACL_CONIG_RULE_KEY.format(acl_name)) + + context_client = ContextClient() + device_client = DeviceClient() + + _device = context_client.GetDevice(DeviceId(**json_device_id(device_uuid))) + + + for cr in _device.device_config.config_rules: + if cr.WhichOneof('config_rule') == 'custom': + if ep_uuid_match := RE_ACL_CONIG_RULE_KEY.match(cr.custom.resource_key): + endpoint_uuid = ep_uuid_match.groups(0)[0] + resource_value_dict = json.loads(cr.custom.resource_value) + type_str = resource_value_dict['rule_set']['type'] + interface = resource_value_dict['interface'] + break + else: + raise NotFound(f'ACL not found') + + acl_config_rule = ConfigRule() + acl_config_rule.action = ConfigActionEnum.CONFIGACTION_DELETE + acl_config_rule.acl.rule_set.name = acl_name + acl_config_rule.acl.interface = interface + acl_config_rule.acl.rule_set.type = getattr(AclRuleTypeEnum, type_str) + acl_config_rule.acl.endpoint_id.device_id.device_uuid.uuid = device_uuid + acl_config_rule.acl.endpoint_id.endpoint_uuid.uuid = endpoint_uuid + + device = Device() + device.CopyFrom(_device) + del device.device_config.config_rules[:] + device.device_config.config_rules.append(acl_config_rule) + response = device_client.ConfigureDevice(device) + return (response.device_uuid.uuid).strip("\"\n") diff --git a/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_services.py b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_services.py new file mode 100644 index 000000000..2d03e61b6 --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/acl_services.py @@ -0,0 +1,68 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Dict + +from flask import request +from flask_restful import Resource +from werkzeug.exceptions import NotFound + +from common.proto.context_pb2 import Device, DeviceId +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Device import json_device_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient + +from nbi.service.rest_server.nbi_plugins.tools.Authentication import HTTP_AUTH + +from .ietf_acl_parser import config_rule_from_ietf_acl + +LOGGER = logging.getLogger(__name__) + +class ACLs(Resource): + # @HTTP_AUTH.login_required + def get(self): + return {} + + # @HTTP_AUTH.login_required + def post(self, device_uuid: str): + if not request.is_json: + raise UnsupportedMediaType("JSON pyload is required") + request_data: Dict = request.json + LOGGER.debug("Request: {:s}".format(str(request_data))) + attached_interface = request_data["ietf-access-control-list"]["acls"]['attachment-points']['interface'][0]['interface-id'] + + context_client = ContextClient() + device_client = DeviceClient() + + _device = context_client.GetDevice(DeviceId(**json_device_id(device_uuid))) + + for ep in _device.device_endpoints: + if ep.name == attached_interface: + endpoint_uuid = ep.endpoint_id.endpoint_uuid.uuid + break + else: + raise NotFound(f'interface {attached_interface} not found in device {device_uuid}') + + acl_config_rule = config_rule_from_ietf_acl(request_data, device_uuid, endpoint_uuid, sequence_id=1, subinterface=0) + + LOGGER.info(f"ACL Config Rule: {grpc_message_to_json_string(acl_config_rule)}") + + device = Device() + device.CopyFrom(_device) + del device.device_config.config_rules[:] + device.device_config.config_rules.append(acl_config_rule) + response = device_client.ConfigureDevice(device) + return (response.device_uuid.uuid).strip("\"\n") diff --git a/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_parser.py b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_parser.py new file mode 100644 index 000000000..b378153f8 --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_parser.py @@ -0,0 +1,164 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Dict, Optional, TypedDict +from pydantic import BaseModel, Field + +from common.proto.acl_pb2 import AclForwardActionEnum, AclRuleTypeEnum, AclEntry +from common.proto.context_pb2 import ConfigActionEnum, ConfigRule + +class Ipv4(BaseModel): + dscp: int = 0 + source_ipv4_network: str = Field(serialization_alias="source-ipv4-network", default="") + destination_ipv4_network: str = Field(serialization_alias="destination-ipv4-network", default="") + +class Port(BaseModel): + port: int = 0 + operator: str = "eq" + +class Tcp(BaseModel): + flags: str = "" + source_port: Port = Field(serialization_alias="source-port", default_factory=lambda: Port()) + destination_port: Port = Field(serialization_alias="destination-port", default_factory=lambda: Port()) + +class Matches(BaseModel): + ipv4: Ipv4 = Ipv4() + tcp: Tcp = Tcp() + +class Action(BaseModel): + forwarding: str = "" + +class Ace(BaseModel): + name: str = "custom_rule" + matches: Matches = Matches() + actions: Action = Action() + +class Aces(BaseModel): + ace: List[Ace] = [Ace()] + +class Acl(BaseModel): + name: str = "" + type: str = "" + aces: Aces = Aces() + +class Name(BaseModel): + name: str = "" + +class AclSet(BaseModel): + acl_set: List[Name] = Field(serialization_alias="acl-set", default=[Name()]) + +class AclSets(BaseModel): + acl_sets: AclSet = Field(serialization_alias="acl-sets", default=AclSet()) + +class Ingress(BaseModel): + ingress: AclSets = AclSets() + +class Interface(BaseModel): + interface_id: str = Field(serialization_alias="interface-id", default="") + ingress: Ingress = Ingress() + +class Interfaces(BaseModel): + interface: List[Interface] = [Interface()] + +class AttachmentPoints(BaseModel): + attachment_points: Interfaces = Field(serialization_alias="attachment-points", default=Interfaces()) + +class Acls(BaseModel): + acl: List[Acl] = [Acl()] + attachment_points: AttachmentPoints = Field(serialization_alias="attachment-points", default=AttachmentPoints()) + +class IETF_ACL(BaseModel): + acls: Acls = Acls() + + +IETF_TFS_RULE_TYPE_MAPPING = { + "ipv4-acl-type": "ACLRULETYPE_IPV4", + "ipv6-acl-type": "ACLRULETYPE_IPV6", +} + +IETF_TFS_FORWARDING_ACTION_MAPPING = { + "drop": "ACLFORWARDINGACTION_DROP", + "accept": "ACLFORWARDINGACTION_ACCEPT", +} + +TFS_IETF_RULE_TYPE_MAPPING = { + "ACLRULETYPE_IPV4": "ipv4-acl-type", + "ACLRULETYPE_IPV6": "ipv6-acl-type", +} + +TFS_IETF_FORWARDING_ACTION_MAPPING = { + "ACLFORWARDINGACTION_DROP": "drop", + "ACLFORWARDINGACTION_ACCEPT": "accept", +} + +def config_rule_from_ietf_acl( + request: Dict, + device_uuid: str, + endpoint_uuid: str, + sequence_id: int, + subinterface: int, +) -> ConfigRule: + the_acl = request["ietf-access-control-list"]["acls"]["acl"][0] + acl_ip_data = the_acl["aces"]["ace"][0]["matches"]["ipv4"] + acl_tcp_data = the_acl["aces"]["ace"][0]["matches"]["tcp"] + attachemnt_interface = request["ietf-access-control-list"]["acls"]['attachment-points']['interface'][0] + source_address = acl_ip_data["source-ipv4-network"] + destination_address = acl_ip_data["destination-ipv4-network"] + source_port = acl_tcp_data['source-port']['port'] + destination_port = acl_tcp_data['destination-port']['port'] + ietf_action = the_acl["aces"]["ace"][0]["actions"]["forwarding"] + interface_id = attachemnt_interface['interface-id'] + + acl_config_rule = ConfigRule() + acl_config_rule.action = ConfigActionEnum.CONFIGACTION_SET + acl_config_rule.acl.interface = interface_id + acl_endpoint_id = acl_config_rule.acl.endpoint_id + acl_endpoint_id.device_id.device_uuid.uuid = device_uuid + acl_endpoint_id.endpoint_uuid.uuid = endpoint_uuid + acl_rule_set = acl_config_rule.acl.rule_set + acl_rule_set.name = the_acl["name"] + acl_rule_set.type = getattr(AclRuleTypeEnum, IETF_TFS_RULE_TYPE_MAPPING[the_acl['type']]) + acl_rule_set.description = ( + f'{ietf_action} {the_acl["type"]}: {source_address}:{source_port}->{destination_address}:{destination_port}' + ) + acl_entry = AclEntry() + acl_entry.sequence_id = sequence_id + acl_entry.match.src_address = source_address + acl_entry.match.dst_address = destination_address + acl_entry.match.src_port = source_port + acl_entry.match.dst_port = destination_port + acl_entry.match.dscp = acl_ip_data["dscp"] + acl_entry.match.flags = acl_tcp_data["flags"] + acl_entry.action.forward_action = getattr(AclForwardActionEnum, IETF_TFS_FORWARDING_ACTION_MAPPING[ietf_action]) + acl_rule_set.entries.append(acl_entry) + + return acl_config_rule + +def ietf_acl_from_config_rule_resource_value(config_rule_rv: Dict) -> Dict: + rule_set = config_rule_rv['rule_set'] + acl_entry = rule_set['entries'][0] + match_ = acl_entry['match'] + + ipv4 = Ipv4(dscp=match_["dscp"], source_ipv4_network=match_["src_address"], destination_ipv4_network=match_["dst_address"]) + tcp = Tcp(flags=match_["flags"], source_port=Port(port=match_["src_port"]), destination_port=Port(port=match_["dst_port"])) + matches = Matches(ipvr=ipv4, tcp=tcp) + aces = Aces(ace=[Ace(matches=matches, actions=Action(forwarding=TFS_IETF_FORWARDING_ACTION_MAPPING[acl_entry["action"]["forward_action"]]))]) + acl = Acl(name=rule_set["name"], type=TFS_IETF_RULE_TYPE_MAPPING[rule_set["type"]], aces=aces) + acl_sets = AclSets(acl_sets=AclSet(acl_set=[Name(name=rule_set["name"])])) + ingress = Ingress(ingress=acl_sets) + interfaces = Interfaces(interface=[Interface(interface_id=config_rule_rv["interface"], ingress=ingress)]) + acls = Acls(acl=[acl], attachment_points=AttachmentPoints(attachment_points=interfaces)) + ietf_acl = IETF_ACL(acls=acls) + + return ietf_acl.model_dump(by_alias=True) \ No newline at end of file diff --git a/src/nbi/tests/data/ietf_acl.json b/src/nbi/tests/data/ietf_acl.json new file mode 100644 index 000000000..3cbdd0c67 --- /dev/null +++ b/src/nbi/tests/data/ietf_acl.json @@ -0,0 +1,56 @@ +{ + "ietf-access-control-list": { + "acls": { + "acl": [ + { + "name": "sample-ipv4-acl", + "type": "ipv4-acl-type", + "aces": { + "ace": [ + { + "name": "rule1", + "matches": { + "ipv4": { + "dscp": 18, + "source-ipv4-network": "192.168.10.6/24", + "destination-ipv4-network": "192.168.20.6/24" + }, + "tcp": { + "flags": "syn", + "source-port": { + "port": 1444, + "operator": "eq" + }, + "destination-port": { + "port": 1333, + "operator": "eq" + } + } + }, + "actions": { + "forwarding": "drop" + } + } + ] + } + } + ], + "attachment-points": { + "interface": [ + { + "interface-id": "PORT-ce1", + "ingress": { + "acl-sets": { + "acl-set": [ + { + "name": "sample-ipv4-acl" + } + ] + } + } + } + ] + } + } + } +} \ No newline at end of file -- GitLab From 14072dc13e8fae70853fd5edbbfe4c811a322690 Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 12 Mar 2024 12:15:39 +0000 Subject: [PATCH 028/985] ietf acl client for tfs nbi interaction added --- .../nbi_plugins/ietf_acl/ietf_acl_client.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_client.py diff --git a/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_client.py b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_client.py new file mode 100644 index 000000000..79ec388a2 --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/ietf_acl/ietf_acl_client.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +import json +import time + +BASE_URL = "/restconf/data" +POST_URL = "/device={}/ietf-access-control-list:acls" +DELETE_URL = "/device={}/ietf-access-control-list:acl={}" + +class IetfTfsClient: + def __init__(self, + tfs_host: str = "10.1.1.119", + tfs_port: int = 80, + username: str = "admin", + password: str = "admin", + timeout: int = 10, + allow_redirects: bool = True, + ) -> None: + self.host = tfs_host + self.port = tfs_port + self.username = username + self.password = password + self.timeout = timeout + self.allow_redirects = allow_redirects + + def post(self, device_uuid: str, ietf_acl_data: dict) -> str: + request_url = "http://{:s}:{:d}{:s}{:s}".format(self.host, self.port, BASE_URL, POST_URL.format(device_uuid)) + reply = requests.request("post", request_url, timeout=self.timeout, json=ietf_acl_data, allow_redirects=self.allow_redirects) + return reply.text + + def get(self, device_uuid: str, acl_name: str) -> str: + request_url = "http://{:s}:{:d}{:s}{:s}".format(self.host, self.port, BASE_URL, DELETE_URL.format(device_uuid, acl_name)) + reply = requests.request("get", request_url, timeout=self.timeout, allow_redirects=self.allow_redirects) + return reply.text + + def delete(self, device_uuid: str, acl_name: str) -> str: + request_url = "http://{:s}:{:d}{:s}{:s}".format(self.host, self.port, BASE_URL, DELETE_URL.format(device_uuid, acl_name)) + reply = requests.request("delete", request_url, timeout=self.timeout, allow_redirects=self.allow_redirects) + return reply.text + +if __name__ == "__main__": + csg1_device_uuid = 'b71fd62f-e3d4-5956-93b9-3139094836cf' + acl_name = 'sample-ipv4-acl' + acl_request_path = 'src/nbi/tests/data/ietf_acl.json' + with open(acl_request_path, 'r') as afile: + acl_request_data = json.load(afile) + + ietf_tfs_client = IetfTfsClient() + post_response = ietf_tfs_client.post(csg1_device_uuid, acl_request_data) + print(f"post response: {post_response}") + time.sleep(.5) + get_response = ietf_tfs_client.get(csg1_device_uuid, acl_name) + print(f"get response: {get_response}") + time.sleep(.5) + delete_response = ietf_tfs_client.delete(csg1_device_uuid, acl_name) + print(f"delete response: {delete_response}") \ No newline at end of file -- GitLab From ef19aedcd200b0d00a715cc0696c506062a793e6 Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 12 Mar 2024 14:45:51 +0000 Subject: [PATCH 029/985] enabling ipv4 ingress filter for acl support xml in ipinfusion added to acl recipe and xml renderer changed to jinja2 --- .../service/drivers/openconfig/OpenConfigDriver.py | 6 +++--- .../service/drivers/openconfig/templates/__init__.py | 4 ++++ .../acl/interfaces/ingress/enable_ingress_filter.xml | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/enable_ingress_filter.xml diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 99ae1c8db..793b292c7 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -227,10 +227,10 @@ def edit_config( resource_key,resource_value = resource chk_string(str_resource_name + '.key', resource_key, allow_empty=False) - str_config_messages = compose_config( # get template for configuration - resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer='pyangbind') # str_config_messages = compose_config( # get template for configuration - # resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) + # resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer='pyangbind') + str_config_messages = compose_config( # get template for configuration + resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) for str_config_message in str_config_messages: # configuration of the received templates if str_config_message is None: raise UnsupportedResourceKeyException(resource_key) logger.debug('[{:s}] str_config_message[{:d}] = {:s}'.format( diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index dcd3cf683..8df8fa28f 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -118,17 +118,21 @@ def compose_config( # template generation templates =[] if "acl_ruleset" in resource_key: # MANAGING ACLs if True: #vendor == 'ipinfusion': #! ipinfusion proprietary netconf receipe is used temporarily + enable_ingress_filter_path = 'acl/interfaces/ingress/enable_ingress_filter.xml' acl_entry_path = 'acl/acl-set/acl-entry/edit_config_ipinfusion_proprietary.xml' acl_ingress_path = 'acl/interfaces/ingress/edit_config_ipinfusion_proprietary.xml' data : Dict[str, Any] = acl_cr_to_dict_ipinfusion_proprietary(resource_value, delete=delete) else: + enable_ingress_filter_path = 'acl/interfaces/ingress/enable_ingress_filter.xml' acl_entry_path = 'acl/acl-set/acl-entry/edit_config.xml' acl_ingress_path = 'acl/interfaces/ingress/edit_config.xml' data : Dict[str, Any] = acl_cr_to_dict(resource_value, delete=delete) if delete: # unpair acl and interface before removing acl templates.append(JINJA_ENV.get_template(acl_ingress_path)) templates.append(JINJA_ENV.get_template(acl_entry_path)) + templates.append(JINJA_ENV.get_template(enable_ingress_filter_path)) else: + templates.append(JINJA_ENV.get_template(enable_ingress_filter_path)) templates.append(JINJA_ENV.get_template(acl_entry_path)) templates.append(JINJA_ENV.get_template(acl_ingress_path)) else: diff --git a/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/enable_ingress_filter.xml b/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/enable_ingress_filter.xml new file mode 100644 index 000000000..274028657 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/acl/interfaces/ingress/enable_ingress_filter.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file -- GitLab From 582d324933ae6b5acf89c842fd145e4521c045ed Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 12 Mar 2024 16:09:11 +0000 Subject: [PATCH 030/985] removin changes of commit f10241a6 as it is related to MEC-PoC --- .../drivers/openconfig/OpenConfigDriver.py | 3 - .../drivers/openconfig/templates/Tools.py | 3 +- .../VPN/Network_instance_multivendor.py | 36 +---- .../nbi_plugins/etsi_bwm/Resources.py | 15 +- .../rest_server/nbi_plugins/etsi_bwm/Tools.py | 140 +++++------------- .../l3nm_openconfig/ConfigRules.py | 66 ++------- 6 files changed, 53 insertions(+), 210 deletions(-) diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 793b292c7..8c6e07b3f 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -226,9 +226,6 @@ def edit_config( chk_length(str_resource_name, resource, min_length=2, max_length=2) resource_key,resource_value = resource chk_string(str_resource_name + '.key', resource_key, allow_empty=False) - - # str_config_messages = compose_config( # get template for configuration - # resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer='pyangbind') str_config_messages = compose_config( # get template for configuration resource_key, resource_value, delete=delete, vendor=netconf_handler.vendor, message_renderer=netconf_handler.message_renderer) for str_config_message in str_config_messages: # configuration of the received templates diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 3e8043680..79bebef51 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -61,8 +61,7 @@ def generate_templates(resource_key: str, resource_value: str, delete: bool,vend elif "inter_instance_policies" in resource_key: result_templates.append(associate_RP_to_NI(data)) elif "protocols" in resource_key: - result_templates.append(add_protocol_NI(data, vendor, delete)) - # if vendor == "ADVA": result_templates.append(add_protocol_NI(data, vendor, delete)) + if vendor == "ADVA": result_templates.append(add_protocol_NI(data, vendor, delete)) elif "table_connections" in resource_key: result_templates.append(create_table_conns(data, delete)) elif "interface" in resource_key: diff --git a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py index e36955a0d..c4d494ea6 100644 --- a/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py +++ b/src/device/service/drivers/openconfig/templates/VPN/Network_instance_multivendor.py @@ -116,9 +116,6 @@ def add_protocol_NI(parameters,vendor, DEL): else: with tag('network-instance'): with tag('name'):text(parameters['name']) - with tag('config'): - with tag('name'): text(parameters['name']) - with tag('type', 'xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types"'): text('oc-ni-types:DEFAULT_INSTANCE') with tag('protocols'): with tag('protocol'): with tag('identifier', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) @@ -126,41 +123,14 @@ def add_protocol_NI(parameters,vendor, DEL): with tag('config'): with tag('identifier', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) with tag('name') :text(parameters['protocol_name']) - with tag('enabled'): text('true') if "BGP" in parameters['identifier']: with tag('bgp'): with tag('global'): - with tag('afi-safis'): - with tag('afi-safi'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('config'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('enabled'): text('true') with tag('config'): with tag('as') :text(parameters['as']) - with tag('peer-groups'): - with tag('peer-group'): - with tag('peer-group-name'): text('IBGP') - with tag('config'): - with tag('peer-group-name'): text('IBGP') - with tag('peer-as'): text(parameters['protocol_name']) - with tag('afi-safis'): - with tag('afi-safi'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('config'): - with tag('afi-safi-name', 'xmlns:oc-bgp-types="http://openconfig.net/yang/bgp-types"'): text('oc-bgp-types:IPV4_UNICAST') - with tag('enabled'): text('true') - - if 'neighbors' in parameters: - with tag('neighbors'): - for neighbor in parameters['neighbors']: - with tag('neighbor'): - with tag('neighbor-address'): text(neighbor['ip_address']) - with tag('config'): - with tag('neighbor-address'): text(neighbor['ip_address']) - with tag('peer-group'): text('IBGP') - # if vendor == "ADVA": - if True: + if "router-id" in parameters: + with tag('router-id'):text(parameters['router-id']) + if vendor == "ADVA": with tag('tables'): with tag('table'): with tag('protocol', 'xmlns:oc-pol-types="http://openconfig.net/yang/policy-types"'):text('oc-pol-types:',parameters['identifier']) diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py index 394b50de8..3fccbbb55 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Resources.py @@ -13,9 +13,7 @@ # limitations under the License. import copy, deepmerge, json, logging -from typing import Dict from common.Constants import DEFAULT_CONTEXT_NAME -from werkzeug.exceptions import UnsupportedMediaType from context.client.ContextClient import ContextClient from flask_restful import Resource, request from service.client.ServiceClient import ServiceClient @@ -39,20 +37,15 @@ class BwInfo(_Resource): return bw_allocations def post(self): - if not request.is_json: - raise UnsupportedMediaType('JSON payload is required') - request_data: Dict = request.get_json() - service = bwInfo_2_service(self.client, request_data) + bwinfo = request.get_json() + service = bwInfo_2_service(self.client, bwinfo) stripped_service = copy.deepcopy(service) stripped_service.ClearField('service_endpoint_ids') stripped_service.ClearField('service_constraints') stripped_service.ClearField('service_config') - try: - response = format_grpc_to_json(self.service_client.CreateService(stripped_service)) - response = format_grpc_to_json(self.service_client.UpdateService(service)) - except Exception as e: # pylint: disable=broad-except - return e + response = format_grpc_to_json(self.service_client.CreateService(stripped_service)) + response = format_grpc_to_json(self.service_client.UpdateService(service)) return response diff --git a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py index d3be769c9..a78d28193 100644 --- a/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/etsi_bwm/Tools.py @@ -14,35 +14,22 @@ import json import logging -import re import time from decimal import ROUND_HALF_EVEN, Decimal from flask.json import jsonify from common.proto.context_pb2 import ( - ContextId, Empty, EndPointId, ServiceId, ServiceTypeEnum, Service, ServiceStatusEnum, Constraint, Constraint_SLA_Capacity, + ContextId, Empty, EndPointId, ServiceId, ServiceTypeEnum, Service, Constraint, Constraint_SLA_Capacity, ConfigRule, ConfigRule_Custom, ConfigActionEnum) from common.tools.grpc.Tools import grpc_message_to_json -from common.tools.grpc.ConfigRules import update_config_rule_custom from common.tools.object_factory.Context import json_context_id from common.tools.object_factory.Service import json_service_id LOGGER = logging.getLogger(__name__) -ENDPOINT_SETTINGS_KEY = '/device[{:s}]/endpoint[{:s}]/vlan[{:d}]/settings' -RE_CONFIG_RULE_IF_SUBIF = re.compile(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$') -MEC_CONSIDERED_FIELDS = ['requestType', 'sessionFilter', 'fixedAllocation', 'allocationDirection'] -ALLOCATION_DIRECTION_DESCRIPTIONS = { - '00' : 'Downlink (towards the UE)', - '01' : 'Uplink (towards the application/session)', - '10' : 'Symmetrical'} -VLAN_TAG = 0 -PREFIX_LENGTH = 24 -BGP_AS = 65000 -policy_AZ = 'srv_{:d}_a'.format(VLAN_TAG) -policy_ZA = 'srv_{:d}_b'.format(VLAN_TAG) def service_2_bwInfo(service: Service) -> dict: response = {} + # allocationDirection = '??' # String: 00 = Downlink (towards the UE); 01 = Uplink (towards the application/session); 10 = Symmetrical response['appInsId'] = service.service_id.service_uuid.uuid # String: Application instance identifier for constraint in service.service_constraints: if constraint.WhichOneof('constraint') == 'sla_capacity': @@ -68,108 +55,47 @@ def service_2_bwInfo(service: Service) -> dict: return response -def bwInfo_2_service(client, bw_info: dict) -> Service: - # add description to allocationDirection code - if ad_code := bw_info.get('allocationDirection'): - bw_info['allocationDirection'] = {'code': ad_code, 'description': ALLOCATION_DIRECTION_DESCRIPTIONS[ad_code]} - if 'sessionFilter' in bw_info: - bw_info['sessionFilter'] = bw_info['sessionFilter'][0] # Discard other items in sessionFilter field - +def bwInfo_2_service(client, bwInfo: dict) -> Service: service = Service() - - service_config_rules = service.service_config.config_rules - - route_distinguisher = '{:5d}:{:03d}'.format(BGP_AS, VLAN_TAG) - settings_cr_key = '/settings' - settings_cr_value = {'bgp_as':(BGP_AS, True), 'route_distinguisher': (route_distinguisher, True)} - update_config_rule_custom(service_config_rules, settings_cr_key, settings_cr_value) - - request_cr_key = '/request' - request_cr_value = {k:bw_info[k] for k in MEC_CONSIDERED_FIELDS} - - config_rule = ConfigRule() - config_rule.action = ConfigActionEnum.CONFIGACTION_SET - config_rule_custom = ConfigRule_Custom() - config_rule_custom.resource_key = request_cr_key - config_rule_custom.resource_value = json.dumps(request_cr_value) - config_rule.custom.CopyFrom(config_rule_custom) - service_config_rules.append(config_rule) - - if 'sessionFilter' in bw_info: - a_ip = bw_info['sessionFilter']['sourceIp'] - z_ip = bw_info['sessionFilter']['dstAddress'] + for key in ['allocationDirection', 'fixedBWPriority', 'requestType', 'timeStamp', 'sessionFilter']: + if key not in bwInfo: + continue + config_rule = ConfigRule() + config_rule.action = ConfigActionEnum.CONFIGACTION_SET + config_rule_custom = ConfigRule_Custom() + config_rule_custom.resource_key = key + if key != 'sessionFilter': + config_rule_custom.resource_value = str(bwInfo[key]) + else: + config_rule_custom.resource_value = json.dumps(bwInfo[key]) + config_rule.custom.CopyFrom(config_rule_custom) + service.service_config.config_rules.append(config_rule) + + if 'sessionFilter' in bwInfo: + a_ip = bwInfo['sessionFilter'][0]['sourceIp'] + z_ip = bwInfo['sessionFilter'][0]['dstAddress'] devices = client.ListDevices(Empty()).devices - router_id_counter = 1 for device in devices: - device_endpoint_uuids = {ep.name:ep.endpoint_id.endpoint_uuid.uuid for ep in device.device_endpoints} for cr in device.device_config.config_rules: - if cr.WhichOneof('config_rule') != 'custom': - continue - match_subif = RE_CONFIG_RULE_IF_SUBIF.match(cr.custom.resource_key) - if not match_subif: - continue - address_ip = json.loads(cr.custom.resource_value).get('address_ip') - if address_ip not in [a_ip, z_ip]: - continue - port_name = 'PORT-' + match_subif.groups(0)[0] # `PORT-` added as prefix - ep_id = EndPointId() - ep_id.endpoint_uuid.uuid = device_endpoint_uuids[port_name] - ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid - service.service_endpoint_ids.append(ep_id) - - # add interface config rules - endpoint_settings_key = ENDPOINT_SETTINGS_KEY.format(device.name, port_name, VLAN_TAG) - if address_ip == a_ip: - field_updates = { - 'address_ip': (address_ip, True), - # 'router_id': ('.'.join([str(router_id_counter)]*4), True), - 'router_id': ('200.1.1.1', True), - 'neighbor_address_ip': ('192.168.150.2', True), - 'route_distinguisher': (route_distinguisher, True), - 'sub_interface_index': (0, True), - 'vlan_id' : (VLAN_TAG, True), - # 'bgp_as': (BGP_AS+router_id_counter, True), - 'bgp_as': (BGP_AS, True), - 'ip_address': (address_ip, True), - 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (policy_AZ, True), - 'policy_ZA' : (policy_ZA, True), - 'address_prefix' : (PREFIX_LENGTH, True), - } - elif address_ip == z_ip: - field_updates = { - 'address_ip': (address_ip, True), - # 'router_id': ('.'.join([str(router_id_counter)]*4), True), - 'router_id': ('200.1.1.2', True), - 'neighbor_address_ip': ('192.168.150.1', True), - 'route_distinguisher': (route_distinguisher, True), - 'sub_interface_index': (0, True), - 'vlan_id' : (VLAN_TAG, True), - # 'bgp_as': (BGP_AS+router_id_counter, True), - 'bgp_as': (BGP_AS, True), - 'ip_address': (address_ip, True), - 'prefix_length': (PREFIX_LENGTH, True), - 'policy_AZ' : (policy_ZA, True), - 'policy_ZA' : (policy_AZ, True), - 'address_prefix' : (PREFIX_LENGTH, True), - } - router_id_counter += 1 - LOGGER.debug(f'BEFORE UPDATE -> device.device_config.config_rules: {service_config_rules}') - update_config_rule_custom(service_config_rules, endpoint_settings_key, field_updates) - LOGGER.debug(f'AFTER UPDATE -> device.device_config.config_rules: {service_config_rules}') - - service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED + if cr.WhichOneof('config_rule') == 'custom' and cr.custom.resource_key == '_connect/settings': + for ep in json.loads(cr.custom.resource_value)['endpoints']: + if 'ip' in ep and (ep['ip'] == a_ip or ep['ip'] == z_ip): + ep_id = EndPointId() + ep_id.endpoint_uuid.uuid = ep['uuid'] + ep_id.device_id.device_uuid.uuid = device.device_id.device_uuid.uuid + service.service_endpoint_ids.append(ep_id) + service.service_type = ServiceTypeEnum.SERVICETYPE_L3NM - if 'appInsId' in bw_info: - service.service_id.service_uuid.uuid = bw_info['appInsId'] + if 'appInsId' in bwInfo: + service.service_id.service_uuid.uuid = bwInfo['appInsId'] service.service_id.context_id.context_uuid.uuid = 'admin' - service.name = bw_info['appInsId'] + service.name = bwInfo['appInsId'] - if 'fixedAllocation' in bw_info: + if 'fixedAllocation' in bwInfo: capacity = Constraint_SLA_Capacity() - capacity.capacity_gbps = float(bw_info['fixedAllocation']) / 1.e9 + capacity.capacity_gbps = float(bwInfo['fixedAllocation']) / 1.e9 constraint = Constraint() constraint.sla_capacity.CopyFrom(capacity) service.service_constraints.append(constraint) diff --git a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py index 0369d6207..1e4425cdb 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l3nm_openconfig/ConfigRules.py @@ -37,7 +37,6 @@ def setup_config_rules( vlan_id = json_endpoint_settings.get('vlan_id', 1 ) # 400 address_ip = json_endpoint_settings.get('address_ip', '0.0.0.0') # '2.2.2.1' address_prefix = json_endpoint_settings.get('address_prefix', 24 ) # 30 - neighbor_address_ip = json_endpoint_settings.get('neighbor_address_ip', '0.0.0.0') # '2.2.2.1' policy_import = json_endpoint_settings.get('policy_AZ', '2' ) # 2 policy_export = json_endpoint_settings.get('policy_ZA', '7' ) # 30 @@ -47,21 +46,18 @@ def setup_config_rules( network_subinterface_desc = json_endpoint_settings.get('subif_description','') #service_short_uuid = service_uuid.split('-')[-1] #network_instance_name = '{:s}-NetInst'.format(service_short_uuid) - # network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 - network_instance_name = 'default' + network_instance_name = json_endpoint_settings.get('ni_name', service_uuid.split('-')[-1]) #ELAN-AC:1 - ''' - # if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) - if_subif_name = '{:s}'.format(endpoint_name[5:]) + if_subif_name = '{:s}.{:d}'.format(endpoint_name, vlan_id) json_config_rules = [ - # # Configure Interface (not used) - # json_config_rule_set( + # Configure Interface (not used) + #json_config_rule_set( # '/interface[{:s}]'.format(endpoint_name), { # 'name': endpoint_name, # 'description': network_interface_desc, # 'mtu': mtu, - # }), + #}), #Create network instance json_config_rule_set( @@ -78,10 +74,8 @@ def setup_config_rules( json_config_rule_set( '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { 'name': network_instance_name, - # 'protocol_name': 'BGP', - 'protocol_name': bgp_as, + 'protocol_name': 'BGP', 'identifier': 'BGP', - # 'identifier': bgp_as, 'as': bgp_as, 'router_id': router_id, }), @@ -107,8 +101,7 @@ def setup_config_rules( json_config_rule_set( '/interface[{:s}]/subinterface[{:d}]'.format(if_subif_name, sub_interface_index), { 'name' : if_subif_name, - # 'type' :'l3ipvlan', - 'type' :'ethernetCsmacd', + 'type' :'l3ipvlan', 'mtu' : mtu, 'index' : sub_interface_index, 'description' : network_subinterface_desc, @@ -190,40 +183,6 @@ def setup_config_rules( }), ] - ''' - if_subif_name = '{:s}'.format(endpoint_name[5:]) - - json_config_rules = [ - - #Add DIRECTLY CONNECTED protocol to network instance - json_config_rule_set( - '/network_instance[{:s}]/protocols[DIRECTLY_CONNECTED]'.format(network_instance_name), { - 'name': network_instance_name, - 'identifier': 'DIRECTLY_CONNECTED', - 'protocol_name': 'DIRECTLY_CONNECTED', - }), - - # Add BGP neighbors - json_config_rule_set( - '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), { - 'name': network_instance_name, - 'protocol_name': bgp_as, - 'identifier': 'BGP', - 'as': bgp_as, - 'router_id': router_id, - 'neighbors': [{'ip_address': neighbor_address_ip, 'remote_as': bgp_as}] - }), - json_config_rule_set( - '/network_instance[{:s}]/table_connections[DIRECTLY_CONNECTED][BGP][IPV4]'.format(network_instance_name), { - 'name' : network_instance_name, - 'src_protocol' : 'DIRECTLY_CONNECTED', - 'dst_protocol' : 'BGP', - 'address_family' : 'IPV4', - 'default_import_policy': 'ACCEPT_ROUTE', - }), - - ] - for res_key, res_value in endpoint_acls: json_config_rules.append( {'action': 1, 'acl': res_value} @@ -242,8 +201,7 @@ def teardown_config_rules( json_endpoint_settings : Dict = endpoint_settings.value service_short_uuid = service_uuid.split('-')[-1] - # network_instance_name = '{:s}-NetInst'.format(service_short_uuid) - network_instance_name = json_endpoint_settings.get('ni_name', service_short_uuid) #ELAN-AC:1 + network_instance_name = '{:s}-NetInst'.format(service_short_uuid) #network_interface_desc = '{:s}-NetIf'.format(service_uuid) #network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid) @@ -304,10 +262,10 @@ def teardown_config_rules( #Delete interface; automatically deletes: # - /interface[]/subinterface[] - # json_config_rule_delete('/interface[{:s}]/subinterface[0]'.format(if_subif_name), - # { - # 'name': if_subif_name, - # }), + json_config_rule_delete('/interface[{:s}]/subinterface[0]'.format(if_subif_name), + { + 'name': if_subif_name, + }), #Delete network instance; automatically deletes: # - /network_instance[]/interface[] -- GitLab From 7def79c1209173d0be26bf9dc7e493ad8e544fda Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 15:01:11 +0100 Subject: [PATCH 031/985] Slices with IETF format --- src/common/tools/descriptor/Loader.py | 97 +++++++++++++++++++++------ src/common/tools/descriptor/Tools.py | 13 +--- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 24f45aeb6..b2eb75045 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -105,9 +105,69 @@ class DescriptorLoader: self.__devices = self.__descriptors.get('devices' , []) self.__links = self.__descriptors.get('links' , []) self.__services = self.__descriptors.get('services' , []) - self.__slices = self.__descriptors.get('data', []) #Coge de la file el campo slices + self.__slices = self.__descriptors.get('slices' , []) + self.__slices = self.__descriptors.get('ietf-network-slice-service:network-slice-services', {}) self.__connections = self.__descriptors.get('connections', []) + if self.__slices: + json_out = {"slices": [ + { + "slice_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "slice_uuid": {} + }, + "name": {}, + "slice_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/settings", "resource_value": { + "address_families": ["IPV4"], "bgp_as": 65000, "bgp_route_target": "65000:333", "mtu": 1512 + }}} + ]}, + "slice_constraints": [ + {"sla_capacity": {"capacity_gbps": 20.0}}, + {"sla_availability": {"availability": 20.0, "num_disjoint_paths": 1, "all_active": True}}, + {"sla_isolation": {"isolation_level": [0]}} + ], + "slice_endpoint_ids": [ + + ], + "slice_status": {"slice_status": 1} + } + ]} + + for slice_service in self.__slices["slice-service"]: + for slice in json_out["slices"]: + slice["slice_id"]["slice_uuid"] = { "uuid": slice_service["id"]} + slice["name"] = slice_service["description"] + sdp = slice_service["sdps"]["sdp"] + for elemento in sdp: + attcircuits = elemento["attachment-circuits"]["attachment-circuit"] + for attcircuit in attcircuits: + resource_key = "/device[{sdp_id}]/endpoint[{endpoint_id}]/settings".format(sdp_id = elemento["id"], endpoint_id = attcircuit["ac-tp-id"]) + + for tag in attcircuit['ac-tags']['ac-tag']: + if tag.get('tag-type') == 'ietf-nss:vlan-id': + vlan_id = tag.get('value') + else: + vlan_id = 0 + + slice["slice_config"]["config_rules"].append( {"action": 1, "custom": {"resource_key": resource_key, "resource_value": { + "router_id": elemento.get("node-id",[]), "sub_interface_index": 0, "vlan_id": vlan_id + }}}) + slice["slice_endpoint_ids"].append({ + "device_id": {"device_uuid": {"uuid": elemento["id"]}}, + "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_uuid": {"uuid": "admin"}} + }) + slice["slice_constraints"].append({"endpoint_location": { + "endpoint_id": {"device_id": {"device_uuid": {"uuid": elemento["id"]}}, "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}}, + "location": {"region": "4"} + }}) + + # Convertir a JSON de salida + #json_output = json.dumps(json_out, indent=2) + self.__slices = json_out.get('slices' , []) + self.__contexts_add = None self.__topologies_add = None self.__devices_add = None @@ -216,7 +276,6 @@ class DescriptorLoader: # Format CustomConfigRules in Devices, Services and Slices provided in JSON format self.__devices = [format_device_custom_config_rules (device ) for device in self.__devices ] self.__services = [format_service_custom_config_rules(service) for service in self.__services] - LOGGERS.INFO(self.__slices) self.__slices = [format_slice_custom_config_rules (slice_ ) for slice_ in self.__slices ] # Context and Topology require to create the entity first, and add devices, links, services, @@ -237,8 +296,7 @@ class DescriptorLoader: self.__ctx_cli.connect() self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) - self._process_descr('controller', 'add', self.__ctx_cli.SetDevice, Device, controllers ) - self._process_descr('device', 'add', self.__ctx_cli.SetDevice, Device, network_devices ) + self._process_descr('device', 'add', self.__ctx_cli.SetDevice, Device, self.__devices ) self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) self._process_descr('service', 'add', self.__ctx_cli.SetService, Service, self.__services ) self._process_descr('slice', 'add', self.__ctx_cli.SetSlice, Slice, self.__slices ) @@ -266,29 +324,24 @@ class DescriptorLoader: self.__services_add = get_descriptors_add_services(self.__services) self.__slices_add = get_descriptors_add_slices(self.__slices) - controllers_add, network_devices_add = split_controllers_and_network_devices(self.__devices_add) - self.__ctx_cli.connect() self.__dev_cli.connect() self.__svc_cli.connect() self.__slc_cli.connect() - self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) - self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) - self._process_descr('controller', 'add', self.__dev_cli.AddDevice, Device, controllers_add ) - self._process_descr('device', 'add', self.__dev_cli.AddDevice, Device, network_devices_add ) - self._process_descr('device', 'config', self.__dev_cli.ConfigureDevice, Device, self.__devices_config) - self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) - self._process_descr('service', 'add', self.__svc_cli.CreateService, Service, self.__services_add ) - self._process_descr('service', 'update', self.__svc_cli.UpdateService, Service, self.__services ) - self._process_descr('slice', 'add', self.__slc_cli.CreateSlice, Slice, self.__slices_add ) - self._process_descr('slice', 'update', self.__slc_cli.UpdateSlice, Slice, self.__slices ) - - # By default the Context component automatically assigns devices and links to topologies based on their - # endpoints, and assigns topologies, services, and slices to contexts based on their identifiers. - - # The following statement is useless; up to now, any use case requires assigning a topology, service, or - # slice to a different context. + self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) + self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) + self._process_descr('device', 'add', self.__dev_cli.AddDevice, Device, self.__devices_add ) + self._process_descr('device', 'config', self.__dev_cli.ConfigureDevice, Device, self.__devices_config) + self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) + self._process_descr('service', 'add', self.__svc_cli.CreateService, Service, self.__services_add ) + self._process_descr('service', 'update', self.__svc_cli.UpdateService, Service, self.__services ) + self._process_descr('slice', 'add', self.__slc_cli.CreateSlice, Slice, self.__slices_add ) + self._process_descr('slice', 'update', self.__slc_cli.UpdateSlice, Slice, self.__slices ) + + # Update context and topology is useless: + # - devices and links are assigned to topologies automatically by Context component + # - topologies, services, and slices are assigned to contexts automatically by Context component #self._process_descr('context', 'update', self.__ctx_cli.SetContext, Context, self.__contexts ) # In some cases, it might be needed to assign devices and links to multiple topologies; the diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index 11e79211f..ad1c402a0 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -16,7 +16,6 @@ import copy, json from typing import Dict, List, Optional, Tuple, Union from common.DeviceTypes import DeviceTypeEnum -#context es la db, al inicio esta vacía def get_descriptors_add_contexts(contexts : List[Dict]) -> List[Dict]: contexts_add = copy.deepcopy(contexts) for context in contexts_add: @@ -54,13 +53,11 @@ def get_descriptors_add_slices(slices : List[Dict]) -> List[Dict]: TypeResourceValue = Union[str, int, bool, float, dict, list] def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: for config_rule in config_rules: - # if 'custom' not in config_rule: continue #suponemos que siempre son custom, quitamos esta linea + if 'custom' not in config_rule: continue custom_resource_value : TypeResourceValue = config_rule['custom']['resource_value'] if isinstance(custom_resource_value, (dict, list)): custom_resource_value = json.dumps(custom_resource_value, sort_keys=True, indent=0) config_rule['custom']['resource_value'] = custom_resource_value - elif not isinstance(custom_resource_value, str): - config_rule['custom']['resource_value'] = str(custom_resource_value) return config_rules def format_device_custom_config_rules(device : Dict) -> Dict: @@ -75,14 +72,10 @@ def format_service_custom_config_rules(service : Dict) -> Dict: service['service_config']['config_rules'] = config_rules return service -#UTILIZA LA FUNCION FORMAT_CUSTOM_CONFIG_RULES -#cambio def format_slice_custom_config_rules(slice_ : Dict) -> Dict: - #donde cojo los config_rules - #las config_rules parecen estar en ACs? - config_rules = slice_.get('sdps', []) + config_rules = slice_.get('slice_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - slice_['sdps']['sdp']['attachment-circuits'] = config_rules + slice_['slice_config']['config_rules'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: -- GitLab From 02d28cbc1cc70a1356edc13a8b6319159d9c97b4 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:33:28 +0100 Subject: [PATCH 032/985] code cleanup --- manifests/sliceservice.yaml | 2 +- .../drivers/openconfig/templates/Inventory.py | 33 +-------------- .../drivers/openconfig/templates/Tools.py | 5 --- src/slice/service/SliceServiceServicerImpl.py | 42 ++++++++----------- .../service/slice_grouper/SliceGrouper.py | 32 +++++++------- src/webui/service/main/routes.py | 4 +- 6 files changed, 36 insertions(+), 82 deletions(-) diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index 61f5b1d21..e7e5c1604 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: SLICE_GROUPING value: "DISABLE" envFrom: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index e45958538..65562bc5b 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -15,7 +15,7 @@ import logging, lxml.etree as ET from typing import Any, Dict, List, Tuple from .Namespace import NAMESPACES -from .Tools import add_value_from_tag, add_int_from_tag +from .Tools import add_value_from_tag LOGGER = logging.getLogger(__name__) @@ -55,8 +55,6 @@ XPATH_PORTS = "//ocp:components/ocp:component" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] parent_types = {} - #Initialized count to 0 for index - count = 0 for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} @@ -65,7 +63,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: inventory['class'] = '' inventory['attributes'] = {} component_reference = [] - component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue @@ -85,34 +82,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue - - ##Added (after the checking of the name and the class) - #Physical index- Index of the component in the array - - add_int_from_tag(inventory['attributes'], 'physical-index', count) - count +=1 - - ##Added - #FRU - if inventory['class'] == 'FRU': - component_isfru = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) - add_value_from_tag(inventory['attributes'], 'isfru', component_isfru) - ##ID - component_id = xml_component.find('ocp:state/ocp:id', namespaces=NAMESPACES) - if not component_id is None: - add_value_from_tag(inventory['attributes'], 'id', component_id) - - ##OPER_STATUS - component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) - if not component_oper_status is None: - add_value_from_tag(inventory['attributes'], 'oper-status', component_oper_status) - - ##MODEL_ID - component_model_id = xml_component.find('ocp:state/ocp:entity-id', namespaces=NAMESPACES) - if not component_model_id is None: - add_value_from_tag(inventory['attributes'], 'model-id', component_model_id) - - ## component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 78e61e0ae..79bebef51 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -26,11 +26,6 @@ def add_value_from_tag(target : Dict, field_name: str, field_value : ET.Element, if cast is not None: field_value = cast(field_value) target[field_name] = field_value -def add_int_from_tag(target : Dict, field_name: str, field_value : int, cast=None) -> None: - if field_value is None: return - if cast is not None: field_value = cast(field_value) - target[field_name] = field_value - def add_value_from_collection(target : Dict, field_name: str, field_value : Collection) -> None: if field_value is None or len(field_value) == 0: return target[field_name] = field_value diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index 52552a6ed..8a834f352 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -#agrupar slices agrupar recursos para no hacer mas configs - from typing import Optional import grpc, json, logging #, deepdiff from common.proto.context_pb2 import ( @@ -32,14 +30,13 @@ from interdomain.client.InterdomainClient import InterdomainClient from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper -LOGGER = logging.getLogger(__name__) #crea un objeto de registro con el nombre del modulo actual - +LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('Slice', 'RPC') -class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio gRPC definido por SliceServiceServicer +class SliceServiceServicerImpl(SliceServiceServicer): def __init__(self): LOGGER.debug('Creating Servicer...') - self._slice_grouper = SliceGrouper() #crea una instancia de slicegrouper + self._slice_grouper = SliceGrouper() LOGGER.debug('Servicer Created') def create_update(self, request : Slice) -> SliceId: @@ -48,9 +45,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g context_client = ContextClient() slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req - slice_rw = Slice() #crea nueva slice desde la slice de la req + slice_rw = Slice() slice_rw.CopyFrom(request if slice_ro is None else slice_ro) - if len(request.name) > 0: slice_rw.name = request.name #actualizamos el nombre y estado de la slice rw + if len(request.name) > 0: slice_rw.name = request.name slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member @@ -59,7 +56,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member - slice_id_with_uuids = context_client.SetSlice(slice_rw) #actualizar o crear la slice en la db + slice_id_with_uuids = context_client.SetSlice(slice_rw) if len(slice_rw.slice_endpoint_ids) < 2: # pylint: disable=no-member # unable to identify the kind of slice; just update endpoints, constraints and config rules @@ -68,18 +65,17 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g reply = context_client.SetSlice(slice_rw) context_client.close() return reply - #si tiene menos de 2 endpoints se omite la actualizacion y se retorna el sliceid - - slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) #obtenemos la slice actualizada - + + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) + #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) #LOGGER.info('json_updated_slice = {:s}'.format(str(json_updated_slice))) #changes = deepdiff.DeepDiff(json_current_slice, json_updated_slice) #LOGGER.info('changes = {:s}'.format(str(changes))) - if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): #si la slice es interdominio - interdomain_client = InterdomainClient() #que es interdomain client? + if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): + interdomain_client = InterdomainClient() slice_id = interdomain_client.RequestSlice(slice_with_uuids) slice_ = context_client.GetSlice(slice_id) slice_active = Slice() @@ -101,10 +97,10 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g service_client = ServiceClient() try: - _service = context_client.GetService(service_id) #obtener info de un servicio si existe + _service = context_client.GetService(service_id) except: # pylint: disable=bare-except # pylint: disable=no-member - service_request = Service() # sino se crea un nuevo servicio + service_request = Service() service_request.service_id.CopyFrom(service_id) service_request.service_type = ServiceTypeEnum.SERVICETYPE_UNKNOWN service_request.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED @@ -113,7 +109,6 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g service_request = Service() service_request.CopyFrom(_service) -#actualiza el servicio con la info de la slice # pylint: disable=no-member copy_endpoint_ids(request.slice_endpoint_ids, service_request.service_endpoint_ids) copy_constraints(request.slice_constraints, service_request.service_constraints) @@ -167,11 +162,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g slice_active.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_ACTIVE # pylint: disable=no-member context_client.SetSlice(slice_active) - service_client.close() #liberar recursos, que es realmente? - context_client.close() #db teraflow + service_client.close() + context_client.close() return slice_id - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) #agrega funcionalidades de metrica y seguridad + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: #try: # slice_ = context_client.GetSlice(request.slice_id) @@ -201,7 +196,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteSlice(self, request : SliceId, context : grpc.ServicerContext) -> Empty: - context_client = ContextClient() #coge la info de una slice + context_client = ContextClient() try: _slice = context_client.GetSlice(request) except: # pylint: disable=bare-except @@ -210,11 +205,8 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g _slice_rw = Slice() _slice_rw.CopyFrom(_slice) - #cambia el status _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) -#elimina la slice considerando si es interdominio o no, y desagrupa la slice eliminada - #elimina los servicios asociados a la slice if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index d59531a1b..11aa9bb58 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -14,7 +14,7 @@ import logging, pandas, threading from typing import Dict, Optional, Tuple -from sklearn.cluster import KMeans #algoritmo de agrupamiento de scikit-learn (biblio de aprendizaje automatico) +from sklearn.cluster import KMeans from common.proto.context_pb2 import Slice from common.tools.grpc.Tools import grpc_message_to_json_string from .Constants import SLICE_GROUPS @@ -27,30 +27,30 @@ LOGGER = logging.getLogger(__name__) class SliceGrouper: def __init__(self) -> None: - self._lock = threading.Lock() #controla el acceso concurrente - self._is_enabled = is_slice_grouping_enabled() #esta habilitado el agrupamiento de slices? + self._lock = threading.Lock() + self._is_enabled = is_slice_grouping_enabled() LOGGER.info('Slice Grouping: {:s}'.format('ENABLED' if self._is_enabled else 'DISABLED')) if not self._is_enabled: return - metrics_exporter = MetricsExporter() #instancia de la clase + metrics_exporter = MetricsExporter() metrics_exporter.create_table() - self._slice_groups = create_slice_groups(SLICE_GROUPS) #grupos de slices + self._slice_groups = create_slice_groups(SLICE_GROUPS) # Initialize and fit K-Means with the pre-defined clusters we want, i.e., one per slice group - df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) #data frame con info de los grupos - k_means = KMeans(n_clusters=df_groups.shape[0]) #modelo utilizado para el agrupamiento + df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) + k_means = KMeans(n_clusters=df_groups.shape[0]) k_means.fit(df_groups[['availability', 'capacity_gbps']]) df_groups['label'] = k_means.predict(df_groups[['availability', 'capacity_gbps']]) self._k_means = k_means self._df_groups = df_groups - self._group_mapping : Dict[str, Dict] = { #Dict = dictionary + self._group_mapping : Dict[str, Dict] = { group['name']:{k:v for k,v in group.items() if k != 'name'} - for group in list(df_groups.to_dict('records')) #mapeo de nombres de grupo a sus atributos - } + for group in list(df_groups.to_dict('records')) + } - label_to_group = {} #mapeo de etiquetas a nombres de grupo + label_to_group = {} for group_name,group_attrs in self._group_mapping.items(): label = group_attrs['label'] availability = group_attrs['availability'] @@ -60,7 +60,7 @@ class SliceGrouper: label_to_group[label] = group_name self._label_to_group = label_to_group - def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: #selecciona un grupo para una slice + def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: with self._lock: grouping_parameters = get_slice_grouping_parameters(slice_obj) LOGGER.debug('[_select_group] grouping_parameters={:s}'.format(str(grouping_parameters))) @@ -78,16 +78,16 @@ class SliceGrouper: return group_name, availability, capacity_gbps @property - def is_enabled(self): return self._is_enabled #indica si el agrupamiento de slices esta habilitado - - def group(self, slice_obj : Slice) -> bool: #determina el grupo al que debe pertenecer la slice + def is_enabled(self): return self._is_enabled + + def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[group] selected_group={:s}'.format(str(selected_group))) if selected_group is None: return False return add_slice_to_group(slice_obj, selected_group) - def ungroup(self, slice_obj : Slice) -> bool: # desagrupa la slice de un grupo + def ungroup(self, slice_obj : Slice) -> bool: LOGGER.debug('[ungroup] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[ungroup] selected_group={:s}'.format(str(selected_group))) diff --git a/src/webui/service/main/routes.py b/src/webui/service/main/routes.py index eba758ff3..75f036bef 100644 --- a/src/webui/service/main/routes.py +++ b/src/webui/service/main/routes.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64, json, logging -import traceback #, re +import base64, json, logging #, re from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request from common.proto.context_pb2 import ContextList, Empty, TopologyId, TopologyList from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications @@ -114,7 +113,6 @@ def home(): except Exception as e: # pylint: disable=broad-except LOGGER.exception('Descriptor load failed') flash(f'Descriptor load failed: `{str(e)}`', 'danger') - traceback.print_exc() # Agregar esta línea para imprimir el traceback completo finally: context_client.close() device_client.close() -- GitLab From 008be1bab8dcafdf7fe95231135546ee00f602d3 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:37:57 +0100 Subject: [PATCH 033/985] code cleanup --- .../service/drivers/openconfig/templates/Inventory.py | 1 - src/slice/service/SliceServiceServicerImpl.py | 9 +++++---- src/slice/service/slice_grouper/SliceGrouper.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 65562bc5b..e2999c579 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -82,7 +82,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue - component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index 8a834f352..cbe2dd5c7 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -31,6 +31,7 @@ from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper LOGGER = logging.getLogger(__name__) + METRICS_POOL = MetricsPool('Slice', 'RPC') class SliceServiceServicerImpl(SliceServiceServicer): @@ -43,7 +44,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is # being modified. context_client = ContextClient() - slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req + slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) slice_rw = Slice() slice_rw.CopyFrom(request if slice_ro is None else slice_ro) @@ -51,7 +52,6 @@ class SliceServiceServicerImpl(SliceServiceServicer): slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member -#copiamos endpoints, reglas y configuraciones de la req a la slice copy_endpoint_ids(request.slice_endpoint_ids, slice_rw.slice_endpoint_ids ) # pylint: disable=no-member copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member @@ -65,9 +65,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): reply = context_client.SetSlice(slice_rw) context_client.close() return reply - + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) - + #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) #LOGGER.info('json_updated_slice = {:s}'.format(str(json_updated_slice))) @@ -207,6 +207,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): _slice_rw.CopyFrom(_slice) _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) + if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 11aa9bb58..66d293e1e 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -48,7 +48,7 @@ class SliceGrouper: self._group_mapping : Dict[str, Dict] = { group['name']:{k:v for k,v in group.items() if k != 'name'} for group in list(df_groups.to_dict('records')) - } + } label_to_group = {} for group_name,group_attrs in self._group_mapping.items(): @@ -79,7 +79,6 @@ class SliceGrouper: @property def is_enabled(self): return self._is_enabled - def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) -- GitLab From 75ddb3d9e93e9b8656af1e4fd71d32461f1542b1 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:40:41 +0100 Subject: [PATCH 034/985] code cleanup --- src/device/service/drivers/openconfig/templates/Inventory.py | 1 + src/slice/service/slice_grouper/SliceGrouper.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index e2999c579..916af0478 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -82,6 +82,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 66d293e1e..2f1a79181 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -79,6 +79,7 @@ class SliceGrouper: @property def is_enabled(self): return self._is_enabled + def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) -- GitLab From 78ebaf0fa77e9a4f6991a5aa9934efed7c89125a Mon Sep 17 00:00:00 2001 From: armingol Date: Mon, 18 Mar 2024 14:40:57 +0100 Subject: [PATCH 035/985] Add interface information to network instance --- .../openconfig/templates/NetworkInstances.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/device/service/drivers/openconfig/templates/NetworkInstances.py b/src/device/service/drivers/openconfig/templates/NetworkInstances.py index c00995b3a..9de16d0e9 100644 --- a/src/device/service/drivers/openconfig/templates/NetworkInstances.py +++ b/src/device/service/drivers/openconfig/templates/NetworkInstances.py @@ -23,6 +23,8 @@ XPATH_NETWORK_INSTANCES = "//ocni:network-instances/ocni:network-instance" XPATH_NI_PROTOCOLS = ".//ocni:protocols/ocni:protocol" XPATH_NI_TABLE_CONNECTS = ".//ocni:table-connections/ocni:table-connection" +XPATH_NI_INTERFACE = ".//ocni:interfaces/ocni:interface" + XPATH_NI_IIP_AP = ".//ocni:inter-instance-policies/ocni:apply-policy" XPATH_NI_IIP_AP_IMPORT = ".//ocni:config/ocni:import-policy" XPATH_NI_IIP_AP_EXPORT = ".//ocni:config/ocni:export-policy" @@ -136,6 +138,21 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: table_connection['address_family']) response.append((resource_key, table_connection)) + for xml_interface in xml_network_instance.xpath(XPATH_NI_INTERFACE, namespaces=NAMESPACES): + LOGGER.info('xml_interfaces = {:s}'.format(str(ET.tostring(xml_interface)))) + + interface = {} + name_iface = xml_interface.find('ocni:config/ocni:interface', namespaces=NAMESPACES) + if name_iface is None or name_iface.text is None: continue + add_value_from_tag(interface, 'name_iface', name_iface) + + name_subiface = xml_interface.find('ocni:config/ocni:subinterface', namespaces=NAMESPACES) + add_value_from_tag(interface, 'name_subiface', name_subiface) + + resource_key = '/network_instance[{:s}]/interface[{:s}]'.format( + network_instance['name'], interface['name_iface']) + response.append((resource_key, interface)) + for xml_iip_ap in xml_network_instance.xpath(XPATH_NI_IIP_AP, namespaces=NAMESPACES): #LOGGER.info('xml_iip_ap = {:s}'.format(str(ET.tostring(xml_iip_ap)))) -- GitLab From 87e30be2bab5855e2a0c65dd73fc0f0324f3ce92 Mon Sep 17 00:00:00 2001 From: armingol Date: Mon, 18 Mar 2024 14:43:41 +0100 Subject: [PATCH 036/985] code cleanup --- src/common/tools/descriptor/Loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index b2eb75045..5875094d8 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -255,7 +255,7 @@ class DescriptorLoader: _slices = {} for slice_ in self.__slices: context_uuid = slice_['slice_id']['context_id']['context_uuid']['uuid'] - _slices.setdefault(context_uuid, []).append(slice_) #no tenemos context_uuid en este formato, lo meto a mano? + _slices.setdefault(context_uuid, []).append(slice_) return _slices @property -- GitLab From 5f563662ab123c7732cf8f15ccb3b40b65d038ee Mon Sep 17 00:00:00 2001 From: armingol Date: Mon, 18 Mar 2024 14:45:12 +0100 Subject: [PATCH 037/985] code cleanup --- src/common/tools/descriptor/Loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 5875094d8..92390f63e 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -163,9 +163,6 @@ class DescriptorLoader: "endpoint_id": {"device_id": {"device_uuid": {"uuid": elemento["id"]}}, "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}}, "location": {"region": "4"} }}) - - # Convertir a JSON de salida - #json_output = json.dumps(json_out, indent=2) self.__slices = json_out.get('slices' , []) self.__contexts_add = None -- GitLab From c67611f30b5fce1d88e718e89288810a7c5a6e73 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:07:36 +0000 Subject: [PATCH 038/985] New kpi_manger.proto file is created. --- proto/kpi_manager.proto | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 proto/kpi_manager.proto diff --git a/proto/kpi_manager.proto b/proto/kpi_manager.proto new file mode 100644 index 000000000..f5769ed37 --- /dev/null +++ b/proto/kpi_manager.proto @@ -0,0 +1,47 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_manager; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service KpiManagerService{ + rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final + rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final + rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + repeated KpiId kpi_id_list = 3; + kpi_sample_types.KpiSampleType kpi_sample_type = 4; + context.DeviceId device_id = 5; + context.EndPointId endpoint_id = 6; + context.ServiceId service_id = 7; + context.SliceId slice_id = 8; + context.ConnectionId connection_id = 9; + context.LinkId link_id = 10; +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} \ No newline at end of file -- GitLab From 5a85e8f320c5b0e9492436c2a9c09f6f8ba25074 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:08:52 +0000 Subject: [PATCH 039/985] imports are updated to refer to kpi_manager.proto --- proto/device.proto | 7 ++++--- proto/optical_attack_detector.proto | 5 +++-- proto/policy_condition.proto | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/proto/device.proto b/proto/device.proto index 30e60079d..98cca8ce9 100644 --- a/proto/device.proto +++ b/proto/device.proto @@ -16,7 +16,8 @@ syntax = "proto3"; package device; import "context.proto"; -import "monitoring.proto"; +//import "monitoring.proto"; +import "kpi_manager.proto"; service DeviceService { rpc AddDevice (context.Device ) returns (context.DeviceId ) {} @@ -27,8 +28,8 @@ service DeviceService { } message MonitoringSettings { - monitoring.KpiId kpi_id = 1; - monitoring.KpiDescriptor kpi_descriptor = 2; + kpi_manager.KpiId kpi_id = 1; + kpi_manager.KpiDescriptor kpi_descriptor = 2; float sampling_duration_s = 3; float sampling_interval_s = 4; } diff --git a/proto/optical_attack_detector.proto b/proto/optical_attack_detector.proto index ebe3b5e06..0d3ed58de 100644 --- a/proto/optical_attack_detector.proto +++ b/proto/optical_attack_detector.proto @@ -17,7 +17,8 @@ syntax = "proto3"; package optical_attack_detector; import "context.proto"; -import "monitoring.proto"; +//import "monitoring.proto"; +import "kpi_manager.proto"; service OpticalAttackDetectorService { @@ -28,5 +29,5 @@ service OpticalAttackDetectorService { message DetectionRequest { context.ServiceId service_id = 1; - monitoring.KpiId kpi_id = 2; + kpi_manager.KpiId kpi_id = 2; } diff --git a/proto/policy_condition.proto b/proto/policy_condition.proto index 2037af93c..c0af929ef 100644 --- a/proto/policy_condition.proto +++ b/proto/policy_condition.proto @@ -16,10 +16,11 @@ syntax = "proto3"; package policy; import "monitoring.proto"; +import "kpi_manager.proto"; // Condition message PolicyRuleCondition { - monitoring.KpiId kpiId = 1; + kpi_manager.KpiId kpiId = 1; NumericalOperator numericalOperator = 2; monitoring.KpiValue kpiValue = 3; } -- GitLab From a78bcae188ec444a5e78f8a30df6488814d26176 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:10:14 +0000 Subject: [PATCH 040/985] few methods and messages are moved to kpi_manager.ptoto --- proto/monitoring.proto | 58 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) mode change 100644 => 100755 proto/monitoring.proto diff --git a/proto/monitoring.proto b/proto/monitoring.proto old mode 100644 new mode 100755 index 45ba48b02..2706988aa --- a/proto/monitoring.proto +++ b/proto/monitoring.proto @@ -16,13 +16,14 @@ syntax = "proto3"; package monitoring; import "context.proto"; -import "kpi_sample_types.proto"; +import "kpi_manager.proto"; +//import "kpi_sample_types.proto"; service MonitoringService { - rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final - rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final - rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final - rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final +// rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final +// rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final +// rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final +// rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final rpc IncludeKpi (Kpi ) returns (context.Empty ) {} // Stable and final rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {} // Stable and final rpc QueryKpiData (KpiQuery ) returns (RawKpiTable ) {} // Not implemented @@ -35,36 +36,25 @@ service MonitoringService { rpc GetAlarmDescriptor (AlarmID ) returns (AlarmDescriptor ) {} // Stable and final rpc GetAlarmResponseStream(AlarmSubscription ) returns (stream AlarmResponse) {} // Not Stable not final rpc DeleteAlarm (AlarmID ) returns (context.Empty ) {} // Stable and final - rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final - rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final +// rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final +// rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final } -message KpiDescriptor { - KpiId kpi_id = 1; - string kpi_description = 2; - repeated KpiId kpi_id_list = 3; - kpi_sample_types.KpiSampleType kpi_sample_type = 4; - context.DeviceId device_id = 5; - context.EndPointId endpoint_id = 6; - context.ServiceId service_id = 7; - context.SliceId slice_id = 8; - context.ConnectionId connection_id = 9; - context.LinkId link_id = 10; -} + message MonitorKpiRequest { - KpiId kpi_id = 1; + kpi_manager.KpiId kpi_id = 1; float monitoring_window_s = 2; float sampling_rate_s = 3; // Pending add field to reflect Available Device Protocols } message KpiQuery { - repeated KpiId kpi_ids = 1; - float monitoring_window_s = 2; - uint32 last_n_samples = 3; // used when you want something like "get the last N many samples - context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" - context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" + repeated kpi_manager.KpiId kpi_ids = 1; + float monitoring_window_s = 2; + uint32 last_n_samples = 3; // used when you want something like "get the last N many samples + context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" } @@ -74,20 +64,18 @@ message RawKpi { // cell } message RawKpiList { // column - KpiId kpi_id = 1; - repeated RawKpi raw_kpis = 2; + kpi_manager.KpiId kpi_id = 1; + repeated RawKpi raw_kpis = 2; } message RawKpiTable { // table repeated RawKpiList raw_kpi_lists = 1; } -message KpiId { - context.Uuid kpi_id = 1; -} + message Kpi { - KpiId kpi_id = 1; + kpi_manager.KpiId kpi_id = 1; context.Timestamp timestamp = 2; KpiValue kpi_value = 3; } @@ -117,13 +105,11 @@ message KpiList { repeated Kpi kpi = 1; } -message KpiDescriptorList { - repeated KpiDescriptor kpi_descriptor_list = 1; -} + message SubsDescriptor{ SubscriptionID subs_id = 1; - KpiId kpi_id = 2; + kpi_manager.KpiId kpi_id = 2; float sampling_duration_s = 3; float sampling_interval_s = 4; context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time" @@ -148,7 +134,7 @@ message AlarmDescriptor { AlarmID alarm_id = 1; string alarm_description = 2; string name = 3; - KpiId kpi_id = 4; + kpi_manager.KpiId kpi_id = 4; KpiValueRange kpi_value_range = 5; context.Timestamp timestamp = 6; } -- GitLab From c02883ed78e500104a4f57cdbd2f9aa9f7315cc4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:11:13 +0000 Subject: [PATCH 041/985] service name enum and default grpc port is added --- src/common/Constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/Constants.py b/src/common/Constants.py index 30aa09b4c..838b028a7 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -43,6 +43,7 @@ class ServiceNameEnum(Enum): ZTP = 'ztp' POLICY = 'policy' MONITORING = 'monitoring' + KPIMANGER = 'kpiManager' DLT = 'dlt' NBI = 'nbi' CYBERSECURITY = 'cybersecurity' @@ -73,6 +74,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ZTP .value : 5050, ServiceNameEnum.POLICY .value : 6060, ServiceNameEnum.MONITORING .value : 7070, + ServiceNameEnum.KPIMANGER .value : 7071, ServiceNameEnum.DLT .value : 8080, ServiceNameEnum.NBI .value : 9090, ServiceNameEnum.L3_CAD .value : 10001, -- GitLab From 65fd027c946f5e266311877f0541e030867fd1ee Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:11:59 +0000 Subject: [PATCH 042/985] initial client file for kpi manager --- src/kpi_manager/client/KpiManagerClient.py | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/kpi_manager/client/KpiManagerClient.py diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py new file mode 100644 index 000000000..3aa6fc65d --- /dev/null +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -0,0 +1,76 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.proto.context_pb2 import Empty +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList, +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class KpiManagerclient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.KPIMANGER) # update enum + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # update enum + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = KpiManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def SetKpi(self, request : KpiDescriptor) -> KpiId: + LOGGER.debug('SetKpi: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SetKpi(request) + LOGGER.debug('SetKpi result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteKpi(self,request : KpiId) -> Empty: + LOGGER.debug('DeleteKpi: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteKpi(request) + LOGGER.info('DeleteKpi result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetKpiDescriptor(self, request : KpiId) -> KpiDescriptor: + LOGGER.debug('GetKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetKpiDescriptor(request) + LOGGER.debug('GetKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetKpiDescriptorList(self, request : Empty) -> KpiDescriptorList: + LOGGER.debug('GetKpiDescriptorList: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetKpiDescriptorList(request) + LOGGER.debug('GetKpiDescriptorList result: {:s}'.format(grpc_message_to_json_string(response))) + return response \ No newline at end of file -- GitLab From 0b5fd79d8270475cf0e38cc3970ad77a14e8a08e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:13:09 +0000 Subject: [PATCH 043/985] initial service file for kpi manager --- src/kpi_manager/server/KpiManagerServer.py | 122 +++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/kpi_manager/server/KpiManagerServer.py diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py new file mode 100644 index 000000000..0a8932aab --- /dev/null +++ b/src/kpi_manager/server/KpiManagerServer.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from monitoring.service.NameMapping import NameMapping + +from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList +from monitoring.service import ManagementDBTools + +from common.tools.service.GenericGrpcService import GenericGrpcService + +LOGGER = logging.getLogger(__name__) + +METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") +METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT") +METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT") +METRICSDB_TABLE_MONITORING_KPIS = os.environ.get("METRICSDB_TABLE_MONITORING_KPIS") + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServer(KpiManagerServiceServicer): + def __init__(self, cls_name: str = __name__): + LOGGER.info('Init KpiManagerService') + port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # port updated + GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + def install_servicers(self): + # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class + # implementation exists in the same class. + add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From 5b6399dc2bb9665bbcfc4207239923f82549d938 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:36:20 +0000 Subject: [PATCH 044/985] TYPO - "KPIMANGER" changed to "KPIMANAGER" --- src/common/Constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Constants.py b/src/common/Constants.py index 838b028a7..4a0f3a226 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -43,7 +43,7 @@ class ServiceNameEnum(Enum): ZTP = 'ztp' POLICY = 'policy' MONITORING = 'monitoring' - KPIMANGER = 'kpiManager' + KPIMANAGER = 'kpiManager' DLT = 'dlt' NBI = 'nbi' CYBERSECURITY = 'cybersecurity' -- GitLab From 67146f1e6061f3190659ef1d0f87e07b270c746c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:37:23 +0000 Subject: [PATCH 045/985] updated KPIMANAGER ServiceEnumName --- src/kpi_manager/client/KpiManagerClient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index 3aa6fc65d..d31a8b60f 100644 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -29,8 +29,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class KpiManagerclient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.KPIMANGER) # update enum - if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # update enum + if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) # update enum + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # update enum self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) self.channel = None -- GitLab From 2223326625edeeab4a89e0535ce490e73725db1d Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 19 Mar 2024 07:38:50 +0000 Subject: [PATCH 046/985] updation of KPIMANAGER ServiceEnumName and removal of unnecssary variables. --- src/kpi_manager/server/KpiManagerServer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py index 0a8932aab..d42ce14eb 100644 --- a/src/kpi_manager/server/KpiManagerServer.py +++ b/src/kpi_manager/server/KpiManagerServer.py @@ -29,17 +29,12 @@ from common.tools.service.GenericGrpcService import GenericGrpcService LOGGER = logging.getLogger(__name__) -METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") -METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT") -METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT") -METRICSDB_TABLE_MONITORING_KPIS = os.environ.get("METRICSDB_TABLE_MONITORING_KPIS") - METRICS_POOL = MetricsPool('Monitoring', 'RPC') class KpiManagerServer(KpiManagerServiceServicer): def __init__(self, cls_name: str = __name__): LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANGER) # port updated + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed # Init sqlite monitoring db -- GitLab From 5ae5b5479580345b5a2b42b49bc2fb5544d88a48 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 21 Mar 2024 10:58:38 +0100 Subject: [PATCH 047/985] Logical Inventory: First version 1) Modify device component to store ACL data 2) New WebUI tab with logical inventory --- .../drivers/openconfig/templates/Acl.py | 64 +-- src/webui/service/device/routes.py | 10 + src/webui/service/templates/device/home.html | 9 + .../service/templates/device/logical.html | 397 ++++++++++++++++++ 4 files changed, 449 insertions(+), 31 deletions(-) create mode 100644 src/webui/service/templates/device/logical.html diff --git a/src/device/service/drivers/openconfig/templates/Acl.py b/src/device/service/drivers/openconfig/templates/Acl.py index c316772a5..e9a9119c5 100644 --- a/src/device/service/drivers/openconfig/templates/Acl.py +++ b/src/device/service/drivers/openconfig/templates/Acl.py @@ -20,7 +20,7 @@ from .Tools import add_value_from_tag LOGGER = logging.getLogger(__name__) XPATH_ACL_SET = "//ocacl:acl/ocacl:acl-sets/ocacl:acl-set" -XPATH_A_ACL_ENTRY = ".//ocacl:acl-entries/ocacl:ecl-entry" +XPATH_A_ACL_ENTRY = ".//ocacl:acl-entries/ocacl:acl-entry" XPATH_A_IPv4 = ".//ocacl:ipv4/ocacl:config" XPATH_A_TRANSPORT = ".//ocacl:transport/ocacl:config" XPATH_A_ACTIONS = ".//ocacl:actions/ocacl:config" @@ -34,29 +34,31 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] acl = {} + name = {} for xml_acl in xml_data.xpath(XPATH_ACL_SET, namespaces=NAMESPACES): #LOGGER.info('xml_acl = {:s}'.format(str(ET.tostring(xml_acl)))) acl_name = xml_acl.find('ocacl:name', namespaces=NAMESPACES) if acl_name is None or acl_name.text is None: continue - add_value_from_tag(acl, 'name', acl_name) + add_value_from_tag(name, 'name', acl_name) acl_type = xml_acl.find('ocacl:type', namespaces=NAMESPACES) add_value_from_tag(acl, 'type', acl_type) for xml_acl_entries in xml_acl.xpath(XPATH_A_ACL_ENTRY, namespaces=NAMESPACES): - acl_id = xml_acl_entries.find('ocacl:sequence_id', namespaces=NAMESPACES) - add_value_from_tag(acl, 'sequence_id', acl_id) + acl_id = xml_acl_entries.find('ocacl:sequence-id', namespaces=NAMESPACES) + add_value_from_tag(acl, 'sequence-id', acl_id) + LOGGER.info('xml_acl_id = {:s}'.format(str(ET.tostring(acl_id)))) for xml_ipv4 in xml_acl_entries.xpath(XPATH_A_IPv4, namespaces=NAMESPACES): - ipv4_source = xml_ipv4.find('ocacl:source_address', namespaces=NAMESPACES) - add_value_from_tag(acl, 'source_address' , ipv4_source) + ipv4_source = xml_ipv4.find('ocacl:source-address', namespaces=NAMESPACES) + add_value_from_tag(acl, 'source-address' , ipv4_source) - ipv4_destination = xml_ipv4.find('ocacl:destination_address', namespaces=NAMESPACES) - add_value_from_tag(acl, 'destination_address' , ipv4_destination) + ipv4_destination = xml_ipv4.find('ocacl:destination-address', namespaces=NAMESPACES) + add_value_from_tag(acl, 'destination-address' , ipv4_destination) ipv4_protocol = xml_ipv4.find('ocacl:protocol', namespaces=NAMESPACES) add_value_from_tag(acl, 'protocol' , ipv4_protocol) @@ -64,30 +66,30 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: ipv4_dscp = xml_ipv4.find('ocacl:dscp', namespaces=NAMESPACES) add_value_from_tag(acl, 'dscp' , ipv4_dscp) - ipv4_hop_limit = xml_ipv4.find('ocacl:hop_limit', namespaces=NAMESPACES) - add_value_from_tag(acl, 'hop_limit' , ipv4_hop_limit) + ipv4_hop_limit = xml_ipv4.find('ocacl:hop-limit', namespaces=NAMESPACES) + add_value_from_tag(acl, 'hop-limit' , ipv4_hop_limit) for xml_transport in xml_acl_entries.xpath(XPATH_A_TRANSPORT, namespaces=NAMESPACES): - transport_source = xml_transport.find('ocacl:source_port', namespaces=NAMESPACES) - add_value_from_tag(acl, 'source_port' ,transport_source) + transport_source = xml_transport.find('ocacl:source-port', namespaces=NAMESPACES) + add_value_from_tag(acl, 'source-port' ,transport_source) - transport_destination = xml_transport.find('ocacl:destination_port', namespaces=NAMESPACES) - add_value_from_tag(acl, 'destination_port' ,transport_destination) + transport_destination = xml_transport.find('ocacl:destination-port', namespaces=NAMESPACES) + add_value_from_tag(acl, 'destination-port' ,transport_destination) - transport_tcp_flags = xml_transport.find('ocacl:tcp_flags', namespaces=NAMESPACES) - add_value_from_tag(acl, 'tcp_flags' ,transport_tcp_flags) + transport_tcp_flags = xml_transport.find('ocacl:tcp-flags', namespaces=NAMESPACES) + add_value_from_tag(acl, 'tcp-flags' ,transport_tcp_flags) for xml_action in xml_acl_entries.xpath(XPATH_A_ACTIONS, namespaces=NAMESPACES): - action = xml_action.find('ocacl:forwarding_action', namespaces=NAMESPACES) - add_value_from_tag(acl, 'forwarding_action' ,action) + action = xml_action.find('ocacl:forwarding-action', namespaces=NAMESPACES) + add_value_from_tag(acl, 'forwarding-action' ,action) - log_action = xml_action.find('ocacl:log_action', namespaces=NAMESPACES) - add_value_from_tag(acl, 'log_action' ,log_action) + log_action = xml_action.find('ocacl:log-action', namespaces=NAMESPACES) + add_value_from_tag(acl, 'log-action' ,log_action) resource_key = '/acl/acl-set[{:s}][{:s}]/acl-entry[{:s}]'.format( - acl['name'], acl['type'], acl['sequence-id']) + name['name'], acl['type'], acl['sequence-id']) response.append((resource_key,acl)) for xml_interface in xml_data.xpath(XPATH_INTERFACE, namespaces=NAMESPACES): @@ -99,25 +101,25 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: for xml_ingress in xml_interface.xpath(XPATH_I_INGRESS, namespaces=NAMESPACES): - i_name = xml_ingress.find('ocacl:set_name_ingress', namespaces=NAMESPACES) - add_value_from_tag(interface, 'ingress_set_name' , i_name) + i_name = xml_ingress.find('ocacl:set-name-ingress', namespaces=NAMESPACES) + add_value_from_tag(interface, 'ingress-set-name' , i_name) - i_type = xml_ingress.find('ocacl:type_ingress', namespaces=NAMESPACES) - add_value_from_tag(interface, 'ingress_type' , i_type) + i_type = xml_ingress.find('ocacl:type-ingress', namespaces=NAMESPACES) + add_value_from_tag(interface, 'ingress-type' , i_type) resource_key = '/acl/interfaces/ingress[{:s}][{:s}]'.format( - acl['name'], acl['type']) + name['name'], acl['type']) response.append((resource_key,interface)) for xml_egress in xml_interface.xpath(XPATH_I_EGRESS, namespaces=NAMESPACES): - e_name = xml_egress.find('ocacl:set_name_egress', namespaces=NAMESPACES) - add_value_from_tag(interface, 'egress_set_name' , e_name) + e_name = xml_egress.find('ocacl:set-name-egress', namespaces=NAMESPACES) + add_value_from_tag(interface, 'egress-set-name' , e_name) - e_type = xml_egress.find('ocacl:type_egress', namespaces=NAMESPACES) - add_value_from_tag(interface, 'egress_type' , e_type) + e_type = xml_egress.find('ocacl:type-egress', namespaces=NAMESPACES) + add_value_from_tag(interface, 'egress-type' , e_type) resource_key = '/acl/interfaces/egress[{:s}][{:s}]'.format( - acl['name'], acl['type']) + name['name'], acl['type']) response.append((resource_key,interface)) return response diff --git a/src/webui/service/device/routes.py b/src/webui/service/device/routes.py index 8b8bc236a..b579094e3 100644 --- a/src/webui/service/device/routes.py +++ b/src/webui/service/device/routes.py @@ -165,6 +165,16 @@ def inventory(device_uuid: str): context_client.close() return render_template('device/inventory.html', device=device_obj) +@device.route('logical/', methods=['GET', 'POST']) +def logical(device_uuid: str): + context_client.connect() + device_obj = get_device(context_client, device_uuid, rw_copy=False) + if device_obj is None: + flash('Device({:s}) not found'.format(str(device_uuid)), 'danger') + device_obj = Device() + context_client.close() + return render_template('device/logical.html', device=device_obj) + @device.get('/delete') def delete(device_uuid): try: diff --git a/src/webui/service/templates/device/home.html b/src/webui/service/templates/device/home.html index e356fd4fb..b6c50c8dd 100644 --- a/src/webui/service/templates/device/home.html +++ b/src/webui/service/templates/device/home.html @@ -51,6 +51,7 @@ Config Rules + @@ -83,6 +84,14 @@ + + + + + + + + {% endfor %} {% else %} diff --git a/src/webui/service/templates/device/logical.html b/src/webui/service/templates/device/logical.html new file mode 100644 index 000000000..1287c20cf --- /dev/null +++ b/src/webui/service/templates/device/logical.html @@ -0,0 +1,397 @@ + + +{% extends 'base.html' %} + +{% block content %} + + +

Device {{ device.name }} ({{ device.device_id.device_uuid.uuid }})

+ +
+
+ +
+
+
+ +
+
+
    +
  • ACL +
      + {% set acl_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/acl/' in config.custom.resource_key %} + {% if 'acl-set' in config.custom.resource_key %} + {% set acl_name = config.custom.resource_key.split('acl-set[')[1].split('][')[0] %} + {% else %} + {% set acl_name = config.custom.resource_key.split('ress[')[1].split('][')[0] %} + {% endif %} + {% if acl_name|length == 0 %} + {% set acl_name = 'Undefined' %} + {% endif %} + {% if acl_name not in acl_names %} + {% set _ = acl_names.append(acl_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for acl_name in acl_names %} +
    • {{ acl_name }} +
        + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/acl/' in config.custom.resource_key and acl_name in config.custom.resource_key.split('][')[0] %} + {% if 'acl-entry' in config.custom.resource_key %} + {% set rule_number = config.custom.resource_key.split('acl-entry[')[1].split(']')[0] %} +
      • Rule {{ rule_number }}: {{ config.custom.resource_value }}
      • + {% else %} +
      • Interface: {{ config.custom.resource_value }}
      • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
      +
    • + {% endfor %} +
    +
  • +
+ +
    +
  • Routing Policy +
      + {% set pol_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/routing_policy/' in config.custom.resource_key %} + {% if 'policy_definition' in config.custom.resource_key %} + {% set pol_name = config.custom.resource_key.split('policy_definition[')[1].split(']')[0] %} + {% endif %} + {% if pol_name|length == 0 %} + {% set pol_name = 'Undefined' %} + {% endif %} + {% if pol_name not in pol_names %} + {% set _ = pol_names.append(pol_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for pol_name in pol_names %} +
    • {{ pol_name }} +
        + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/routing_policy/' in config.custom.resource_key and pol_name in config.custom.resource_key.split('[')[1].split(']')[0] %} + {% if 'policy_definition' not in config.custom.resource_key %} +
      • {{ config.custom.resource_value }}
      • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
      +
    • + {% endfor %} +
    +
  • +
+ +
    +
  • VRFs +
      +
    • VRF default +
        + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/network_instance' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in 'default' %} + {% if ']/' in config.custom.resource_key%} + {% set aux = config.custom.resource_key.split(']/')[1].split('[')[0] %} +
      • {{ aux.replace('_', ' ').title() }}: {{ config.custom.resource_value }}
      • + {% else %} +
      • Network Instance: {{ config.custom.resource_value }}
      • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
      +
    • + +
    • L3VPN +
        + {% set vpn_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/network_instance' in config.custom.resource_key %} + + {% if 'L3VRF' in config.custom.resource_value %} + {% set vpn_name = config.custom.resource_key.split('network_instance[')[1].split(']')[0] %} + {% if vpn_name not in vpn_names %} + {% set _ = vpn_names.append(vpn_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for vpn_name in vpn_names %} +
      • {{ vpn_name }} +
          + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/network_instance' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in vpn_name %} + {% if ']/' in config.custom.resource_key%} + {% set aux = config.custom.resource_key.split(']/')[1].split('[')[0] %} +
        • {{ aux.replace('_', ' ').title() }}: {{ config.custom.resource_value }}
        • + {% else %} +
        • Network Instance: {{ config.custom.resource_value }}
        • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
        +
      • + {% endfor %} +
      +
    • + +
    • L2VPN +
        + {% set vpn_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/network_instance' in config.custom.resource_key %} + + {% if 'L2VSI' in config.custom.resource_value %} + {% set vpn_name = config.custom.resource_key.split('network_instance[')[1].split(']')[0] %} + {% if vpn_name not in vpn_names %} + {% set _ = vpn_names.append(vpn_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for vpn_name in vpn_names %} +
      • {{ vpn_name }} +
          + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/network_instance' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in vpn_name %} + {% if ']/' in config.custom.resource_key%} + {% set aux = config.custom.resource_key.split(']/')[1].split('[')[0] %} +
        • {{ aux.replace('_', ' ').title() }}: {{ config.custom.resource_value }}
        • + {% else %} +
        • Network Instance: {{ config.custom.resource_value }}
        • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
        +
      • + {% endfor %} +
      +
    • +
    +
  • +
+ +
    +
  • Interfaces +
      +
    • Logical Interfaces +
        + {% set interface_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface[' in config.custom.resource_key %} + {% if 'ethernetCsmacd' in config.custom.resource_value %} + {% set interface_name = config.custom.resource_key.split('interface[')[1].split(']')[0] %} +
      • {{ interface_name}}: {{config.custom.resource_value}}
      • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
      +
    • + +
    • Loopback +
        + {% set interface_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface[' in config.custom.resource_key %} + {% if 'softwareLoopback' in config.custom.resource_value %} + {% set interface_name = config.custom.resource_key.split('interface[')[1].split(']')[0] %} + {% if interface_name not in interface_names %} + {% set _ = interface_names.append(interface_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for interface_name in interface_names %} +
      • {{ interface_name }} +
          + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in interface_name %} + {% if 'subinterface' in config.custom.resource_key %} + {% set subinterface_name = config.custom.resource_key.split('subinterface[')[1].split(']')[0] %} +
        • Subinterface {{subinterface_name}}: {{ config.custom.resource_value }}
        • + {% else %} +
        • {{ config.custom.resource_value }}
        • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
        +
      • + {% endfor %} +
      +
    • + +
    • Interfaces L3 +
        + {% set interface_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface[' in config.custom.resource_key %} + {% if 'l3ipvlan' in config.custom.resource_value %} + {% set interface_name = config.custom.resource_key.split('interface[')[1].split(']')[0] %} + {% if interface_name not in interface_names %} + {% set _ = interface_names.append(interface_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for interface_name in interface_names %} +
      • {{ interface_name }} +
          + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface' in config.custom.resource_key and '/subinterface' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in interface_name %} +
        • {{ config.custom.resource_value }}
        • + {% endif %} + {% endif %} + {% endfor %} +
        +
      • + {% endfor %} +
      +
    • + +
    • Interfaces L2 +
        + {% set interface_names = [] %} + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if '/interface[' in config.custom.resource_key %} + {% if 'l2vlan' in config.custom.resource_value or 'mplsTunnel' in config.custom.resource_value %} + {% set interface_name = config.custom.resource_key.split('interface[')[1].split(']')[0] %} + {% if interface_name not in interface_names %} + {% set _ = interface_names.append(interface_name) %} + {% endif %} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + {% for interface_name in interface_names %} +
      • {{ interface_name }} +
          + {% for config in device.device_config.config_rules %} + {% if config.WhichOneof('config_rule') == 'custom' %} + {% if 'subinterface' in config.custom.resource_key %} + {% if '/interface' in config.custom.resource_key and '/subinterface' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in interface_name %} +
        • {{ config.custom.resource_value }}
        • + {% endif %} + {% else %} + {% if '/interface' in config.custom.resource_key and config.custom.resource_key.split('[')[1].split(']')[0] in interface_name %} +
        • {{ config.custom.resource_value }}
        • + {% endif %} + {% endif %} + {% endif %} + {% endfor %} +
        +
      • + {% endfor %} +
      +
    • +
    +
  • +
+ + +
+
+ +{% endblock %} -- GitLab From e5bfc7b10d25c24d03ecc348d948dba4d0639529 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:21:45 +0000 Subject: [PATCH 048/985] blank line is removed --- src/kpi_manager/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/__init__.py diff --git a/src/kpi_manager/__init__.py b/src/kpi_manager/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From 7ef9ed5c57db3a562f10453958baf18cff6c5ef0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:23:44 +0000 Subject: [PATCH 049/985] Class name is changed from "KpiManagerclient" to "KpiManagerClient" --- src/kpi_manager/client/KpiManagerClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index d31a8b60f..6baca7cef 100644 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -27,7 +27,7 @@ MAX_RETRIES = 15 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') -class KpiManagerclient: +class KpiManagerClient: def __init__(self, host=None, port=None): if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) # update enum if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # update enum -- GitLab From e6b8b7a57ad254ae12bf705238f6b8d6e72c01af Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:24:11 +0000 Subject: [PATCH 050/985] blank line is removed --- src/kpi_manager/service/KpiManagerService.py | 117 +++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/kpi_manager/service/KpiManagerService.py diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py new file mode 100644 index 000000000..d42ce14eb --- /dev/null +++ b/src/kpi_manager/service/KpiManagerService.py @@ -0,0 +1,117 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from monitoring.service.NameMapping import NameMapping + +from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList +from monitoring.service import ManagementDBTools + +from common.tools.service.GenericGrpcService import GenericGrpcService + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServer(KpiManagerServiceServicer): + def __init__(self, cls_name: str = __name__): + LOGGER.info('Init KpiManagerService') + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated + GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + def install_servicers(self): + # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class + # implementation exists in the same class. + add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From 90ffb7281d4db0a987d1902640949ad4153f0462 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:25:57 +0000 Subject: [PATCH 051/985] KpiManager test messages file initial version. --- src/kpi_manager/tests/test_messages.py | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/kpi_manager/tests/test_messages.py diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py new file mode 100644 index 000000000..589d6cb84 --- /dev/null +++ b/src/kpi_manager/tests/test_messages.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.proto import kpi_manager_pb2 +from common.proto.kpi_sample_types_pb2 import KpiSampleType + +def kpi_id(): + _kpi_id = kpi_manager_pb2.KpiId() + _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member + return _kpi_id + +def create_kpi_request(kpi_id_str): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) + _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) + return _create_kpi_request + +def create_kpi_request_b(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member + return _create_kpi_request + +def create_kpi_request_c(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member + return _create_kpi_request + +def create_kpi_request_d(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member + _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member + return _create_kpi_request + +def kpi_descriptor_list(): + _kpi_descriptor_list = kpi_manager_pb2.KpiDescriptorList() + return _kpi_descriptor_list \ No newline at end of file -- GitLab From 7fdc29ac02f3a8dc0f35b5fb1d8cca0af3edf4da Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:28:35 +0000 Subject: [PATCH 052/985] The methods moved to KpiManagerMessages file are removed. --- src/monitoring/tests/Messages.py | 102 +++++++++++++++---------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/src/monitoring/tests/Messages.py b/src/monitoring/tests/Messages.py index a56207d9a..23f4db017 100644 --- a/src/monitoring/tests/Messages.py +++ b/src/monitoring/tests/Messages.py @@ -17,54 +17,54 @@ from common.proto import monitoring_pb2 from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.tools.timestamp.Converters import timestamp_utcnow_to_float -def kpi_id(): - _kpi_id = monitoring_pb2.KpiId() - _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member - return _kpi_id - -def create_kpi_request(kpi_id_str): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) - _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) - return _create_kpi_request - -def create_kpi_request_b(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member - return _create_kpi_request - -def create_kpi_request_c(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member - return _create_kpi_request - -def create_kpi_request_d(): - _create_kpi_request = monitoring_pb2.KpiDescriptor() - _create_kpi_request.kpi_description = 'KPI Description Test' - _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member - _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member - _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member - _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member - _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member - return _create_kpi_request +# def kpi_id(): +# _kpi_id = monitoring_pb2.KpiId() +# _kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member +# return _kpi_id + +# def create_kpi_request(kpi_id_str): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str) +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str) +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str) +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str) +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str) +# return _create_kpi_request + +# def create_kpi_request_b(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC2' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_request_c(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV3' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END3' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON3' # pylint: disable=maybe-no-member +# return _create_kpi_request + +# def create_kpi_request_d(): +# _create_kpi_request = monitoring_pb2.KpiDescriptor() +# _create_kpi_request.kpi_description = 'KPI Description Test' +# _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' # pylint: disable=maybe-no-member +# _create_kpi_request.service_id.service_uuid.uuid = 'SERV4' # pylint: disable=maybe-no-member +# _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC4' # pylint: disable=maybe-no-member +# _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END4' # pylint: disable=maybe-no-member +# _create_kpi_request.connection_id.connection_uuid.uuid = 'CON4' # pylint: disable=maybe-no-member +# return _create_kpi_request def monitor_kpi_request(kpi_uuid, monitoring_window_s, sampling_rate_s): _monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() @@ -80,10 +80,10 @@ def include_kpi_request(kpi_id): _include_kpi_request.kpi_value.floatVal = 500*random() # pylint: disable=maybe-no-member return _include_kpi_request -def kpi_descriptor_list(): - _kpi_descriptor_list = monitoring_pb2.KpiDescriptorList() +# def kpi_descriptor_list(): +# _kpi_descriptor_list = monitoring_pb2.KpiDescriptorList() - return _kpi_descriptor_list +# return _kpi_descriptor_list def kpi_query(kpi_id_list): _kpi_query = monitoring_pb2.KpiQuery() -- GitLab From 7386b351cb42c83fd428d35129dbb44a54e5bbaf Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:29:55 +0000 Subject: [PATCH 053/985] Blank line from the top is removed. --- src/kpi_manager/tests/test_unitary.py | 267 ++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 src/kpi_manager/tests/test_unitary.py diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py new file mode 100644 index 000000000..39d2b2874 --- /dev/null +++ b/src/kpi_manager/tests/test_unitary.py @@ -0,0 +1,267 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, pytest +import logging, json + +from apscheduler.schedulers.background import BackgroundScheduler + +from common.proto.context_pb2 import ConfigActionEnum, Context, ContextId, DeviceOperationalStatusEnum, EventTypeEnum, DeviceEvent, Device, Empty, Topology, TopologyId +from common.Constants import ServiceNameEnum +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME, ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.object_factory.Context import json_context, json_context_id +from common.tools.object_factory.Topology import json_topology, json_topology_id +# from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ +# AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList + + +from device.service.driver_api.DriverFactory import DriverFactory +from device.service.driver_api.DriverInstanceCache import DriverInstanceCache +from device.service.DeviceService import DeviceService +from device.client.DeviceClient import DeviceClient + +from kpi_manager.tests.test_messages import create_kpi_request, create_kpi_request_b, create_kpi_request_c, create_kpi_request_d +# from monitoring.service.MonitoringService import MonitoringService +from kpi_manager.service.KpiManagerService import KpiManagerService +# from monitoring.client.MonitoringClient import MonitoringClient +from kpi_manager.client.KpiManagerClient import KpiManagerClient + +from monitoring.service.ManagementDBTools import ManagementDB +from monitoring.service.MetricsDBTools import MetricsDB +from monitoring.service.NameMapping import NameMapping + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 + +KPIMANAGER_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # avoid privileged ports +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(KPIMANAGER_SERVICE_PORT) + +METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME') + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of Monitoring + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +@pytest.fixture(scope='session') +def context_service(): + LOGGER.info('Initializing MockContextService...') + _service = MockContextService(MOCKSERVICE_PORT) + _service.start() + + LOGGER.info('Yielding MockContextService...') + yield _service + + LOGGER.info('Terminating MockContextService...') + _service.context_servicer.msg_broker.terminate() + _service.stop() + + LOGGER.info('Terminated MockContextService...') + +@pytest.fixture(scope='session') +def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing ContextClient...') + _client = ContextClient() + + LOGGER.info('Yielding ContextClient...') + yield _client + + LOGGER.info('Closing ContextClient...') + _client.close() + + LOGGER.info('Closed ContextClient...') + +@pytest.fixture(scope='session') +def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceService...') + driver_factory = DriverFactory(DRIVERS) + driver_instance_cache = DriverInstanceCache(driver_factory) + _service = DeviceService(driver_instance_cache) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding DeviceService...') + yield _service + + LOGGER.info('Terminating DeviceService...') + _service.stop() + + LOGGER.info('Terminated DeviceService...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +# This fixture will be requested by test cases and last during testing session +@pytest.fixture(scope='session') +def kpi_manager_service( + context_service : MockContextService, # pylint: disable=redefined-outer-name,unused-argument + device_service : DeviceService # pylint: disable=redefined-outer-name,unused-argument + ): + LOGGER.info('Initializing KpiManagerService...') + name_mapping = NameMapping() + # _service = MonitoringService(name_mapping) + _service = KpiManagerService(name_mapping) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerService...') + yield _service + + LOGGER.info('Terminating KpiManagerService...') + _service.stop() + + LOGGER.info('Terminated KpiManagerService...') + +# This fixture will be requested by test cases and last during testing session. +# The client requires the server, so client fixture has the server as dependency. +# def monitoring_client(monitoring_service : MonitoringService): (Add for better understanding) +@pytest.fixture(scope='session') +def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing KpiManagerClient...') + _client = KpiManagerClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerClient...') + yield _client + + LOGGER.info('Closing KpiManagerClient...') + _client.close() + + LOGGER.info('Closed KpiManagerClient...') + +@pytest.fixture(scope='session') +def management_db(): + _management_db = ManagementDB('monitoring.db') + return _management_db + +@pytest.fixture(scope='session') +def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name + return monitoring_service.monitoring_servicer.metrics_db + +# This function os not clear to me (Changes should me made before execution) +@pytest.fixture(scope='session') +def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name + return monitoring_service.monitoring_servicer.metrics_db + #_metrics_db = MetricsDBTools.MetricsDB( + # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) + #return _metrics_db + +@pytest.fixture(scope='session') +def subs_scheduler(): + _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) + _scheduler.start() + return _scheduler + +def ingestion_data(kpi_id_int): + # pylint: disable=redefined-outer-name,unused-argument + metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') + + kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') + for _ in range(50): + kpiSampleType = kpiSampleType_name + kpiId = kpi_id_int + deviceId = 'DEV'+ str(kpi_id_int) + endpointId = 'END' + str(kpi_id_int) + serviceId = 'SERV' + str(kpi_id_int) + sliceId = 'SLC' + str(kpi_id_int) + connectionId = 'CON' + str(kpi_id_int) + time_stamp = timestamp_utcnow_to_float() + kpi_value = 500*random() + + metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, + kpi_value) + sleep(0.1) + +################################################## +# Prepare Environment, should be the first test +################################################## + +def test_prepare_environment( + context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument +): + context_id = json_context_id(DEFAULT_CONTEXT_NAME) + context_client.SetContext(Context(**json_context(DEFAULT_CONTEXT_NAME))) + context_client.SetTopology(Topology(**json_topology(DEFAULT_TOPOLOGY_NAME, context_id=context_id))) + +########################### +# Tests Implementation +########################### + +# Test case that makes use of client fixture to test server's CreateKpi method +def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name + # make call to server + LOGGER.warning('test_create_kpi requesting') + for i in range(3): + response = kpi_manager_client.SetKpi(create_kpi_request(str(i+1))) + LOGGER.debug(str(response)) + assert isinstance(response, KpiId) + +# Test case that makes use of client fixture to test server's DeleteKpi method +def test_delete_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name + # make call to server + LOGGER.warning('delete_kpi requesting') + response = kpi_manager_client.SetKpi(create_kpi_request('4')) + response = kpi_manager_client.DeleteKpi(response) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +# Test case that makes use of client fixture to test server's GetKpiDescriptor method +def test_get_kpi_descriptor_list(kpi_manager_client): # pylint: disable=redefined-outer-name + LOGGER.warning('test_getkpidescritor_kpi begin') + response = kpi_manager_client.GetKpiDescriptorList(Empty()) + LOGGER.debug(str(response)) + assert isinstance(response, KpiDescriptorList) -- GitLab From 124d09dec46e2477c23304605e3628200efc84a7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 08:30:45 +0000 Subject: [PATCH 054/985] File name changed from "KpiManagerServer" to "KpiManagerService" --- src/kpi_manager/server/KpiManagerServer.py | 117 --------------------- 1 file changed, 117 deletions(-) delete mode 100644 src/kpi_manager/server/KpiManagerServer.py diff --git a/src/kpi_manager/server/KpiManagerServer.py b/src/kpi_manager/server/KpiManagerServer.py deleted file mode 100644 index d42ce14eb..000000000 --- a/src/kpi_manager/server/KpiManagerServer.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, os, grpc -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty - -from common.Constants import ServiceNameEnum -from common.Settings import get_service_port_grpc -from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server -from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer -from monitoring.service.NameMapping import NameMapping - -from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList -from monitoring.service import ManagementDBTools - -from common.tools.service.GenericGrpcService import GenericGrpcService - -LOGGER = logging.getLogger(__name__) - -METRICS_POOL = MetricsPool('Monitoring', 'RPC') - -class KpiManagerServer(KpiManagerServiceServicer): - def __init__(self, cls_name: str = __name__): - LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated - GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed - - # Init sqlite monitoring db - self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? - LOGGER.info('MetricsDB initialized --- KPI Manager Service') - - def install_servicers(self): - # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class - # implementation exists in the same class. - add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SetKpi( - self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: - response = KpiId() - kpi_description = request.kpi_description - kpi_sample_type = request.kpi_sample_type - kpi_device_id = request.device_id.device_uuid.uuid - kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid - kpi_service_id = request.service_id.service_uuid.uuid - kpi_slice_id = request.slice_id.slice_uuid.uuid - kpi_connection_id = request.connection_id.connection_uuid.uuid - kpi_link_id = request.link_id.link_uuid.uuid - if request.kpi_id.kpi_id.uuid != "": - response.kpi_id.uuid = request.kpi_id.kpi_id.uuid - # Here the code to modify an existing kpi - else: - data = self.management_db.insert_KPI( - kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, - kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) - response.kpi_id.uuid = str(data) - return response - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: - kpi_id = int(request.kpi_id.uuid) - kpi = self.management_db.get_KPI(kpi_id) - if kpi: - self.management_db.delete_KPI(kpi_id) - else: - LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) - return Empty() - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: - kpi_id = request.kpi_id.uuid - kpi_db = self.management_db.get_KPI(int(kpi_id)) - kpiDescriptor = KpiDescriptor() - if kpi_db is None: - LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) - else: - kpiDescriptor.kpi_description = kpi_db[1] - kpiDescriptor.kpi_sample_type = kpi_db[2] - kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) - kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) - kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) - kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) - kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) - kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) - return kpiDescriptor - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: - kpi_descriptor_list = KpiDescriptorList() - data = self.management_db.get_KPIS() - LOGGER.debug(f"data: {data}") - for item in data: - kpi_descriptor = KpiDescriptor() - kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) - kpi_descriptor.kpi_description = item[1] - kpi_descriptor.kpi_sample_type = item[2] - kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) - kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) - kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) - kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) - kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) - kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) - kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) - return kpi_descriptor_list \ No newline at end of file -- GitLab From 71b8b5362b17711d2a4bc2eeaf0c5c17b7a2b827 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:33:29 +0000 Subject: [PATCH 055/985] imports of KpiId-Descriptor-List is changed from monitoring_pb2 to kpi_manger_pb2 --- src/monitoring/client/MonitoringClient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/monitoring/client/MonitoringClient.py b/src/monitoring/client/MonitoringClient.py index 751ff6e38..493e96ca8 100644 --- a/src/monitoring/client/MonitoringClient.py +++ b/src/monitoring/client/MonitoringClient.py @@ -20,8 +20,9 @@ from common.Settings import get_service_host, get_service_port_grpc from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.monitoring_pb2 import Kpi, KpiDescriptor, KpiId, MonitorKpiRequest, \ - KpiDescriptorList, KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsList, \ +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.proto.monitoring_pb2 import Kpi, MonitorKpiRequest, \ + KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsList, \ SubsResponse, AlarmDescriptor, AlarmID, AlarmList, AlarmResponse, AlarmSubscription, RawKpiTable from common.proto.monitoring_pb2_grpc import MonitoringServiceStub -- GitLab From 3e7573750600b730873138e2a765a0cdabd992a7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:35:10 +0000 Subject: [PATCH 056/985] Spelling correction from "KPIMANGER" to "KPIMANAGER" --- src/common/Constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Constants.py b/src/common/Constants.py index 4a0f3a226..ee737d2bd 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -74,7 +74,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ZTP .value : 5050, ServiceNameEnum.POLICY .value : 6060, ServiceNameEnum.MONITORING .value : 7070, - ServiceNameEnum.KPIMANGER .value : 7071, + ServiceNameEnum.KPIMANAGER .value : 7071, ServiceNameEnum.DLT .value : 8080, ServiceNameEnum.NBI .value : 9090, ServiceNameEnum.L3_CAD .value : 10001, -- GitLab From 9f70a96cf4530bc074f98f4e030e6dca5e94fa57 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 22 Mar 2024 10:35:41 +0000 Subject: [PATCH 057/985] files permission changed --- src/kpi_manager/client/KpiManagerClient.py | 0 src/kpi_manager/service/KpiManagerService.py | 0 src/kpi_manager/tests/test_messages.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/kpi_manager/client/KpiManagerClient.py mode change 100644 => 100755 src/kpi_manager/service/KpiManagerService.py mode change 100644 => 100755 src/kpi_manager/tests/test_messages.py diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py old mode 100644 new mode 100755 diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py old mode 100644 new mode 100755 diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py old mode 100644 new mode 100755 -- GitLab From 95ebbc734aeb400fc7b453878733eb5d7489a963 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:01:45 +0000 Subject: [PATCH 058/985] tests file for Kpi Manager is added --- scripts/run_tests_locally_kpi_manager.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts/run_tests_locally_kpi_manager.sh diff --git a/scripts/run_tests_locally_kpi_manager.sh b/scripts/run_tests_locally_kpi_manager.sh new file mode 100755 index 000000000..eeeec4bb2 --- /dev/null +++ b/scripts/run_tests_locally_kpi_manager.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py \ No newline at end of file -- GitLab From 50bbeb104476a21c4b981ae37ab75566d48438f4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:05:36 +0000 Subject: [PATCH 059/985] Docker build command for Kpi Manager is added and Docker file paths are updated to correct directories. --- src/build.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/build.sh b/src/build.sh index b1a7d299e..9ae91ef10 100755 --- a/src/build.sh +++ b/src/build.sh @@ -18,16 +18,20 @@ cd $(dirname $0) echo "BUILD context" -context/genproto.sh +# context/genproto.sh # genproto.sh file doesn't exist docker build -t "context:develop" -f context/Dockerfile --quiet . -docker build -t "context:test" -f context/tests/Dockerfile --quiet . +# docker build -t "context:test" -f context/tests/Dockerfile --quiet . # Dockerfile doesn't exist -cd monitoring -./genproto.sh -cd .. +# genproto.sh file doesn't exist +# cd monitoring +# ./genproto.sh +# cd .. echo "BUILD monitoring" docker build -t "monitoring:dockerfile" -f monitoring/Dockerfile . +echo "BUILD kpi manager" +docker build -t "kpi_manager:dockerfile" -f kpi_manager/Dockerfile . + echo "Prune unused images" docker image prune --force -- GitLab From e28b5ce2fcde8a28564a864c5d221d4e3e57ed63 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:07:58 +0000 Subject: [PATCH 060/985] Docker name and ports are added according to Kpi Manager Service --- src/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/start.sh b/src/start.sh index 32a016cc0..c3b5d8375 100755 --- a/src/start.sh +++ b/src/start.sh @@ -15,4 +15,4 @@ docker network create -d bridge teraflowbridge -docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge monitoring:dockerfile +docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile -- GitLab From f19ae7505ffeb37b9b4c128ac7440ebe504e8aa7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:09:28 +0000 Subject: [PATCH 061/985] monitoring docker command is added for the reference --- src/start.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/start.sh b/src/start.sh index c3b5d8375..8c3fafe6e 100755 --- a/src/start.sh +++ b/src/start.sh @@ -15,4 +15,5 @@ docker network create -d bridge teraflowbridge -docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile +# docker run -d -p 7070:7070 --name monitoring --network=teraflowbridge monitoring:dockerfile +docker run -d -p 7071:7071 --name kpi_manager --network=teraflowbridge kpi_manager:dockerfile \ No newline at end of file -- GitLab From cb1bddd6a5516fcaa1fb035ec276766f8246bf21 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:10:00 +0000 Subject: [PATCH 062/985] Kpi Manager Service docker file is created --- src/kpi_manager/Dockerfile | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/kpi_manager/Dockerfile diff --git a/src/kpi_manager/Dockerfile b/src/kpi_manager/Dockerfile new file mode 100644 index 000000000..d3d962b9f --- /dev/null +++ b/src/kpi_manager/Dockerfile @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/kpi_manager +WORKDIR /var/teraflow/kpi_manager +COPY src/kpi_manager/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ +COPY src/device/. device/ +COPY src/monitoring/. monitoring/ +COPY src/kpi_manager/. kpi_manager/ + +# Start the service +ENTRYPOINT ["python", "-m", "kpi_manager.service"] -- GitLab From 82cb396dad0a10eda54d73d70155f256b6f1f7e1 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:16:43 +0000 Subject: [PATCH 063/985] temporary requirements file is added. --- src/kpi_manager/requirements.in | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/kpi_manager/requirements.in diff --git a/src/kpi_manager/requirements.in b/src/kpi_manager/requirements.in new file mode 100644 index 000000000..a6183b57e --- /dev/null +++ b/src/kpi_manager/requirements.in @@ -0,0 +1,24 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +anytree==2.8.0 +APScheduler==3.10.1 +influx-line-protocol==0.1.4 +psycopg2-binary==2.9.3 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +questdb==1.0.1 +requests==2.27.1 +xmltodict==0.12.0 +# grpc_health_probe==0.2.0 #getting error on this library \ No newline at end of file -- GitLab From 2d868d953730ab675d629aadb74d69d297c61d89 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:17:13 +0000 Subject: [PATCH 064/985] __init__.py is added is sevice directory --- src/kpi_manager/service/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/service/__init__.py diff --git a/src/kpi_manager/service/__init__.py b/src/kpi_manager/service/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From d1d7b89af69076bad27762048422a573eab6efe0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:17:48 +0000 Subject: [PATCH 065/985] __main__.py file added in Kpi Manager service directory. --- src/kpi_manager/service/__main__.py | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/kpi_manager/service/__main__.py diff --git a/src/kpi_manager/service/__main__.py b/src/kpi_manager/service/__main__.py new file mode 100644 index 000000000..9f0e53246 --- /dev/null +++ b/src/kpi_manager/service/__main__.py @@ -0,0 +1,107 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading, time +from prometheus_client import start_http_server +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, + wait_for_environment_variables) +from common.proto import monitoring_pb2 +from monitoring.service.EventTools import EventsDeviceCollector # import updated +from monitoring.service.NameMapping import NameMapping # import updated +# from .MonitoringService import MonitoringService +from .KpiManagerService import KpiManagerService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def start_kpi_manager(name_mapping : NameMapping): + LOGGER.info('Start Monitoring...',) + + events_collector = EventsDeviceCollector(name_mapping) + events_collector.start() + + # TODO: redesign this method to be more clear and clean + + # Iterate while terminate is not set + while not terminate.is_set(): + list_new_kpi_ids = events_collector.listen_events() + + # Monitor Kpis + if bool(list_new_kpi_ids): + for kpi_id in list_new_kpi_ids: + # Create Monitor Kpi Requests + monitor_kpi_request = monitoring_pb2.MonitorKpiRequest() + monitor_kpi_request.kpi_id.CopyFrom(kpi_id) + monitor_kpi_request.monitoring_window_s = 86400 + monitor_kpi_request.sampling_rate_s = 10 + events_collector._monitoring_client.MonitorKpi(monitor_kpi_request) + + time.sleep(0.5) # let other tasks run; do not overload CPU + else: + # Terminate is set, looping terminates + LOGGER.warning("Stopping execution...") + + events_collector.start() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + # Starting monitoring service + # grpc_service = MonitoringService(name_mapping) + # grpc_service.start() + # start_monitoring(name_mapping) + + grpc_service = KpiManagerService(name_mapping) + grpc_service.start() + + start_kpi_manager(name_mapping) + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) -- GitLab From 343483c835443c26603ca23b152294225fe0fbf3 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:19:02 +0000 Subject: [PATCH 066/985] KpiManagerService file re-structred into generic KpiManagerSevice file. --- src/kpi_manager/service/KpiManagerService.py | 106 ++----------------- 1 file changed, 10 insertions(+), 96 deletions(-) diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py index d42ce14eb..dbbcec2cf 100755 --- a/src/kpi_manager/service/KpiManagerService.py +++ b/src/kpi_manager/service/KpiManagerService.py @@ -12,106 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, os, grpc -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty - from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc +# from common.proto.monitoring_pb2_grpc import add_MonitoringServiceServicer_to_server from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server -from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer -from monitoring.service.NameMapping import NameMapping - -from common.proto.kpi_manager_pb2 import kpiDescriptor, KpiId, KpiDescriptorList -from monitoring.service import ManagementDBTools - from common.tools.service.GenericGrpcService import GenericGrpcService +from kpi_manager.service.KpiManagerServiceServicerImpl import KpiManagerServiceServicerImpl +# from monitoring.service.MonitoringServiceServicerImpl import MonitoringServiceServicerImpl +from monitoring.service.NameMapping import NameMapping -LOGGER = logging.getLogger(__name__) - -METRICS_POOL = MetricsPool('Monitoring', 'RPC') - -class KpiManagerServer(KpiManagerServiceServicer): - def __init__(self, cls_name: str = __name__): - LOGGER.info('Init KpiManagerService') - port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # port updated - GenericGrpcService(port, cls_name = cls_name) # class inheretence was removed - - # Init sqlite monitoring db - self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? - LOGGER.info('MetricsDB initialized --- KPI Manager Service') +class KpiManagerService(GenericGrpcService): + def __init__(self, name_mapping : NameMapping, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) + super().__init__(port, cls_name=cls_name) + self.kpiManagerService_servicer = KpiManagerServiceServicerImpl(name_mapping) def install_servicers(self): - # There is no need to create the "MonitoringServiceServicerImpl" instance because actual class - # implementation exists in the same class. - add_KpiManagerServiceServicer_to_server(KpiManagerServer(), self.server) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SetKpi( - self, request: KpiDescriptor, grpc_context: grpc.ServicerContext - ) -> KpiId: - response = KpiId() - kpi_description = request.kpi_description - kpi_sample_type = request.kpi_sample_type - kpi_device_id = request.device_id.device_uuid.uuid - kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid - kpi_service_id = request.service_id.service_uuid.uuid - kpi_slice_id = request.slice_id.slice_uuid.uuid - kpi_connection_id = request.connection_id.connection_uuid.uuid - kpi_link_id = request.link_id.link_uuid.uuid - if request.kpi_id.kpi_id.uuid != "": - response.kpi_id.uuid = request.kpi_id.kpi_id.uuid - # Here the code to modify an existing kpi - else: - data = self.management_db.insert_KPI( - kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, - kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) - response.kpi_id.uuid = str(data) - return response - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: - kpi_id = int(request.kpi_id.uuid) - kpi = self.management_db.get_KPI(kpi_id) - if kpi: - self.management_db.delete_KPI(kpi_id) - else: - LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) - return Empty() - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: - kpi_id = request.kpi_id.uuid - kpi_db = self.management_db.get_KPI(int(kpi_id)) - kpiDescriptor = KpiDescriptor() - if kpi_db is None: - LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) - else: - kpiDescriptor.kpi_description = kpi_db[1] - kpiDescriptor.kpi_sample_type = kpi_db[2] - kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) - kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) - kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) - kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) - kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) - kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) - return kpiDescriptor - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: - kpi_descriptor_list = KpiDescriptorList() - data = self.management_db.get_KPIS() - LOGGER.debug(f"data: {data}") - for item in data: - kpi_descriptor = KpiDescriptor() - kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) - kpi_descriptor.kpi_description = item[1] - kpi_descriptor.kpi_sample_type = item[2] - kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) - kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) - kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) - kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) - kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) - kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) - kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) - return kpi_descriptor_list \ No newline at end of file + add_KpiManagerServiceServicer_to_server(self.kpiManagerService_servicer, self.server) -- GitLab From 6e9b010e3a52f467f88a1c4c3181c44171295664 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:19:36 +0000 Subject: [PATCH 067/985] KpiManagerServiceImpl file is created. --- .../service/KpiManagerServiceServicerImpl.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/kpi_manager/service/KpiManagerServiceServicerImpl.py diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py new file mode 100644 index 000000000..3a40195da --- /dev/null +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -0,0 +1,104 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from monitoring.service.NameMapping import NameMapping +from monitoring.service import ManagementDBTools + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('Monitoring', 'RPC') + +class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): + def __init__(self, name_mapping : NameMapping): + LOGGER.info('Init KpiManagerService') + + # Init sqlite monitoring db + self.management_db = ManagementDBTools.ManagementDB('monitoring.db') # why monitoring.db here??? + LOGGER.info('MetricsDB initialized --- KPI Manager Service') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpi( + self, request: KpiDescriptor, grpc_context: grpc.ServicerContext + ) -> KpiId: + response = KpiId() + kpi_description = request.kpi_description + kpi_sample_type = request.kpi_sample_type + kpi_device_id = request.device_id.device_uuid.uuid + kpi_endpoint_id = request.endpoint_id.endpoint_uuid.uuid + kpi_service_id = request.service_id.service_uuid.uuid + kpi_slice_id = request.slice_id.slice_uuid.uuid + kpi_connection_id = request.connection_id.connection_uuid.uuid + kpi_link_id = request.link_id.link_uuid.uuid + if request.kpi_id.kpi_id.uuid != "": + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # Here the code to modify an existing kpi + else: + data = self.management_db.insert_KPI( + kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, + kpi_service_id, kpi_slice_id, kpi_connection_id, kpi_link_id) + response.kpi_id.uuid = str(data) + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpi(self, request: KpiId, grpc_context: grpc.ServicerContext) -> Empty: + kpi_id = int(request.kpi_id.uuid) + kpi = self.management_db.get_KPI(kpi_id) + if kpi: + self.management_db.delete_KPI(kpi_id) + else: + LOGGER.info('DeleteKpi error: KpiID({:s}): not found in database'.format(str(kpi_id))) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext) -> KpiDescriptor: + kpi_id = request.kpi_id.uuid + kpi_db = self.management_db.get_KPI(int(kpi_id)) + kpiDescriptor = KpiDescriptor() + if kpi_db is None: + LOGGER.info('GetKpiDescriptor error: KpiID({:s}): not found in database'.format(str(kpi_id))) + else: + kpiDescriptor.kpi_description = kpi_db[1] + kpiDescriptor.kpi_sample_type = kpi_db[2] + kpiDescriptor.device_id.device_uuid.uuid = str(kpi_db[3]) + kpiDescriptor.endpoint_id.endpoint_uuid.uuid = str(kpi_db[4]) + kpiDescriptor.service_id.service_uuid.uuid = str(kpi_db[5]) + kpiDescriptor.slice_id.slice_uuid.uuid = str(kpi_db[6]) + kpiDescriptor.connection_id.connection_uuid.uuid = str(kpi_db[7]) + kpiDescriptor.link_id.link_uuid.uuid = str(kpi_db[8]) + return kpiDescriptor + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptorList(self, request: Empty, grpc_context: grpc.ServicerContext) -> KpiDescriptorList: + kpi_descriptor_list = KpiDescriptorList() + data = self.management_db.get_KPIS() + LOGGER.debug(f"data: {data}") + for item in data: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(item[0]) + kpi_descriptor.kpi_description = item[1] + kpi_descriptor.kpi_sample_type = item[2] + kpi_descriptor.device_id.device_uuid.uuid = str(item[3]) + kpi_descriptor.endpoint_id.endpoint_uuid.uuid = str(item[4]) + kpi_descriptor.service_id.service_uuid.uuid = str(item[5]) + kpi_descriptor.slice_id.slice_uuid.uuid = str(item[6]) + kpi_descriptor.connection_id.connection_uuid.uuid = str(item[7]) + kpi_descriptor.link_id.link_uuid.uuid = str(item[8]) + kpi_descriptor_list.kpi_descriptor_list.append(kpi_descriptor) + return kpi_descriptor_list \ No newline at end of file -- GitLab From ab122c221a6b64283f0afd4b21093e48e1fe54c8 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:32:21 +0000 Subject: [PATCH 068/985] These imports were added temporily --- src/kpi_manager/tests/test_unitary.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 src/kpi_manager/tests/test_unitary.py diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py old mode 100644 new mode 100755 index 39d2b2874..b75ea5751 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# import sys +# sys.path.append('.') import os, pytest import logging, json -- GitLab From 16666d8b744200e973fa30600342a9053aaada65 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 26 Mar 2024 10:34:08 +0000 Subject: [PATCH 069/985] Imports of Kpi, KpiList and KpiDescriptor is changed to kpi_manager_pb2 file. --- src/monitoring/service/MonitoringServiceServicerImpl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/monitoring/service/MonitoringServiceServicerImpl.py b/src/monitoring/service/MonitoringServiceServicerImpl.py index 608b0bad9..e98cfa236 100644 --- a/src/monitoring/service/MonitoringServiceServicerImpl.py +++ b/src/monitoring/service/MonitoringServiceServicerImpl.py @@ -20,8 +20,8 @@ from common.proto.context_pb2 import Empty from common.proto.device_pb2 import MonitoringSettings from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.monitoring_pb2_grpc import MonitoringServiceServicer -from common.proto.monitoring_pb2 import AlarmResponse, AlarmDescriptor, AlarmList, SubsList, KpiId, \ - KpiDescriptor, KpiList, KpiQuery, SubsDescriptor, SubscriptionID, AlarmID, KpiDescriptorList, \ +from common.proto.monitoring_pb2 import AlarmResponse, AlarmDescriptor, AlarmList, SubsList, \ + KpiQuery, SubsDescriptor, SubscriptionID, AlarmID, KpiList,\ MonitorKpiRequest, Kpi, AlarmSubscription, SubsResponse, RawKpiTable, RawKpi, RawKpiList from common.tools.timestamp.Converters import timestamp_string_to_float, timestamp_utcnow_to_float from device.client.DeviceClient import DeviceClient @@ -30,6 +30,8 @@ from monitoring.service.AlarmManager import AlarmManager from monitoring.service.NameMapping import NameMapping from monitoring.service.SubscriptionManager import SubscriptionManager +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList + LOGGER = logging.getLogger(__name__) METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME") -- GitLab From a3ca5899e477d26085cb239f14e5fe1d01fb8803 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:37:57 +0000 Subject: [PATCH 070/985] after temporarily removal of --rcfile file --- scripts/run_tests_locally_kpi_manager.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/run_tests_locally_kpi_manager.sh b/scripts/run_tests_locally_kpi_manager.sh index eeeec4bb2..8ed855a8e 100755 --- a/scripts/run_tests_locally_kpi_manager.sh +++ b/scripts/run_tests_locally_kpi_manager.sh @@ -17,8 +17,12 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src -RCFILE=$PROJECTDIR/coverage/.coveragerc -coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ - kpi_manager/tests/test_unitary.py +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py -# python3 kpi_manager/tests/test_unitary.py \ No newline at end of file +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --verbose \ + kpi_manager/tests/test_unitary.py \ No newline at end of file -- GitLab From 92ea40f1cd78480b7c457444b6a9861c39e935b7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:38:48 +0000 Subject: [PATCH 071/985] removal of unexpected "," --- src/kpi_manager/client/KpiManagerClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py index 6baca7cef..a129cd327 100755 --- a/src/kpi_manager/client/KpiManagerClient.py +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -19,7 +19,7 @@ from common.Settings import get_service_host, get_service_port_grpc from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string from common.proto.context_pb2 import Empty -from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList, +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub LOGGER = logging.getLogger(__name__) -- GitLab From 7b88c37fde4204b2bf73ead2b6f0deb0ab4a7e8a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:39:29 +0000 Subject: [PATCH 072/985] __init__.py is added in kpi_manager/client directory --- src/kpi_manager/client/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/kpi_manager/client/__init__.py diff --git a/src/kpi_manager/client/__init__.py b/src/kpi_manager/client/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/kpi_manager/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From e50c6e937c4832a868dd5851586d175f8d78c638 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 27 Mar 2024 06:40:27 +0000 Subject: [PATCH 073/985] some missing imports were added and some methods are commented temporarily --- src/kpi_manager/tests/test_unitary.py | 92 ++++++++++++++------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/src/kpi_manager/tests/test_unitary.py b/src/kpi_manager/tests/test_unitary.py index b75ea5751..113b26890 100755 --- a/src/kpi_manager/tests/test_unitary.py +++ b/src/kpi_manager/tests/test_unitary.py @@ -17,6 +17,7 @@ # sys.path.append('.') import os, pytest import logging, json +from typing import Union from apscheduler.schedulers.background import BackgroundScheduler @@ -33,6 +34,8 @@ from common.tools.object_factory.Topology import json_topology, json_topology_id # from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \ # AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorList +from common.tools.service.GenericGrpcService import GenericGrpcService +from context.client.ContextClient import ContextClient from device.service.driver_api.DriverFactory import DriverFactory @@ -50,6 +53,9 @@ from monitoring.service.ManagementDBTools import ManagementDB from monitoring.service.MetricsDBTools import MetricsDB from monitoring.service.NameMapping import NameMapping +os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' +from device.service.drivers import DRIVERS + ########################### # Tests Setup ########################### @@ -185,49 +191,49 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab LOGGER.info('Closed KpiManagerClient...') -@pytest.fixture(scope='session') -def management_db(): - _management_db = ManagementDB('monitoring.db') - return _management_db - -@pytest.fixture(scope='session') -def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name - return monitoring_service.monitoring_servicer.metrics_db - -# This function os not clear to me (Changes should me made before execution) -@pytest.fixture(scope='session') -def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name - return monitoring_service.monitoring_servicer.metrics_db - #_metrics_db = MetricsDBTools.MetricsDB( - # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) - #return _metrics_db - -@pytest.fixture(scope='session') -def subs_scheduler(): - _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) - _scheduler.start() - return _scheduler - -def ingestion_data(kpi_id_int): - # pylint: disable=redefined-outer-name,unused-argument - metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') - - kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED - kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') - for _ in range(50): - kpiSampleType = kpiSampleType_name - kpiId = kpi_id_int - deviceId = 'DEV'+ str(kpi_id_int) - endpointId = 'END' + str(kpi_id_int) - serviceId = 'SERV' + str(kpi_id_int) - sliceId = 'SLC' + str(kpi_id_int) - connectionId = 'CON' + str(kpi_id_int) - time_stamp = timestamp_utcnow_to_float() - kpi_value = 500*random() - - metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, - kpi_value) - sleep(0.1) +# @pytest.fixture(scope='session') +# def management_db(): +# _management_db = ManagementDB('monitoring.db') +# return _management_db + +# @pytest.fixture(scope='session') +# def metrics_db(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name +# return monitoring_service.monitoring_servicer.metrics_db + +# # This function os not clear to me (Changes should me made before execution) +# @pytest.fixture(scope='session') +# def metrics_db(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name +# return monitoring_service.monitoring_servicer.metrics_db +# #_metrics_db = MetricsDBTools.MetricsDB( +# # METRICSDB_HOSTNAME, METRICSDB_ILP_PORT, METRICSDB_REST_PORT, METRICSDB_TABLE_MONITORING_KPIS) +# #return _metrics_db + +# @pytest.fixture(scope='session') +# def subs_scheduler(): +# _scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)}) +# _scheduler.start() +# return _scheduler + +# def ingestion_data(kpi_id_int): +# # pylint: disable=redefined-outer-name,unused-argument +# metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring') + +# kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED +# kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '') +# for _ in range(50): +# kpiSampleType = kpiSampleType_name +# kpiId = kpi_id_int +# deviceId = 'DEV'+ str(kpi_id_int) +# endpointId = 'END' + str(kpi_id_int) +# serviceId = 'SERV' + str(kpi_id_int) +# sliceId = 'SLC' + str(kpi_id_int) +# connectionId = 'CON' + str(kpi_id_int) +# time_stamp = timestamp_utcnow_to_float() +# kpi_value = 500*random() + +# metrics_db.write_KPI(time_stamp, kpiId, kpiSampleType, deviceId, endpointId, serviceId, sliceId, connectionId, +# kpi_value) +# sleep(0.1) ################################################## # Prepare Environment, should be the first test -- GitLab From b4b217029b981702d6cccddf9fe1a22f7e4909ed Mon Sep 17 00:00:00 2001 From: mansoca Date: Wed, 27 Mar 2024 11:45:39 +0000 Subject: [PATCH 074/985] Creation VTNManager service --- my_deploy.sh | 3 + src/vnt_manager/.gitlab-ci.yml | 38 ++++++++ src/vnt_manager/Config.py | 13 +++ src/vnt_manager/Dockerfile | 84 ++++++++++++++++ src/vnt_manager/__init__.py | 13 +++ src/vnt_manager/client/VNTManagerClient.py | 71 ++++++++++++++ src/vnt_manager/client/__init__.py | 13 +++ src/vnt_manager/requirements.in | 15 +++ src/vnt_manager/service/VNTManagerService.py | 35 +++++++ .../service/VNTManagerServiceServicerImpl.py | 95 +++++++++++++++++++ src/vnt_manager/service/__init__.py | 13 +++ src/vnt_manager/service/__main__.py | 80 ++++++++++++++++ 12 files changed, 473 insertions(+) create mode 100644 src/vnt_manager/.gitlab-ci.yml create mode 100644 src/vnt_manager/Config.py create mode 100644 src/vnt_manager/Dockerfile create mode 100644 src/vnt_manager/__init__.py create mode 100644 src/vnt_manager/client/VNTManagerClient.py create mode 100644 src/vnt_manager/client/__init__.py create mode 100644 src/vnt_manager/requirements.in create mode 100644 src/vnt_manager/service/VNTManagerService.py create mode 100644 src/vnt_manager/service/VNTManagerServiceServicerImpl.py create mode 100644 src/vnt_manager/service/__init__.py create mode 100644 src/vnt_manager/service/__main__.py diff --git a/my_deploy.sh b/my_deploy.sh index 7dd5e5c3e..212dd7bd6 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -52,6 +52,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate E2E Orchestrator #export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +# Uncomment to activate VNT Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} vnt_manager" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/vnt_manager/.gitlab-ci.yml b/src/vnt_manager/.gitlab-ci.yml new file mode 100644 index 000000000..d1b9da495 --- /dev/null +++ b/src/vnt_manager/.gitlab-ci.yml @@ -0,0 +1,38 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build, tag and push the Docker image to the gitlab registry +build vntmanager: + variables: + IMAGE_NAME: 'vntmanager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml diff --git a/src/vnt_manager/Config.py b/src/vnt_manager/Config.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/Config.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/Dockerfile b/src/vnt_manager/Dockerfile new file mode 100644 index 000000000..8f40741ee --- /dev/null +++ b/src/vnt_manager/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Creating a user for security reasons +RUN groupadd -r teraflow && useradd -u 1001 --no-log-init -r -m -g teraflow teraflow +USER teraflow + +# set working directory +RUN mkdir -p /home/teraflow/controller/common/ +WORKDIR /home/teraflow/controller + +# Get Python packages per module +ENV VIRTUAL_ENV=/home/teraflow/venv +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +COPY --chown=teraflow:teraflow common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /home/teraflow/controller/common +COPY --chown=teraflow:teraflow src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /home/teraflow/controller/common/proto +WORKDIR /home/teraflow/controller/common/proto +RUN touch __init__.py +COPY --chown=teraflow:teraflow proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create module sub-folders +RUN mkdir -p /home/teraflow/controller/vnt_manager +WORKDIR /home/teraflow/controller + +# Get Python packages per module +COPY --chown=teraflow:teraflow ./src/vnt_manager/requirements.in vnt_manager/requirements.in +# consider common and specific requirements to avoid inconsistencies with dependencies +RUN pip-compile --quiet --output-file=vnt_manager/requirements.txt vnt_manager/requirements.in common_requirements.in +RUN python3 -m pip install -r vnt_manager/requirements.txt + +# Add component files into working directory +COPY --chown=teraflow:teraflow ./src/context/. context +COPY --chown=teraflow:teraflow ./src/vnt_manager/. vnt_manager + +# Start the service +ENTRYPOINT ["python", "-m", "vnt_manager.service"] diff --git a/src/vnt_manager/__init__.py b/src/vnt_manager/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/client/VNTManagerClient.py b/src/vnt_manager/client/VNTManagerClient.py new file mode 100644 index 000000000..95db3b6da --- /dev/null +++ b/src/vnt_manager/client/VNTManagerClient.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import grpc + +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceStub +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import delay_exponential, retry +from common.tools.grpc.Tools import grpc_message_to_json +# from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry( + max_retries=MAX_RETRIES, + delay_function=DELAY_FUNCTION, + prepare_method_name="connect", +) + + +class VNTManagerClient: + def __init__(self, host=None, port=None): + if not host: + host = get_service_host(ServiceNameEnum.VNTMANAGER) + if not port: + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + self.endpoint = "{:s}:{:s}".format(str(host), str(port)) + LOGGER.debug("Creating channel to {:s}...".format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug("Channel created") + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = VNTManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: + self.channel.close() + self.channel = None + self.stub = None + + """ + @RETRY_DECORATOR + def Compute(self, request: E2EOrchestratorRequest) -> E2EOrchestratorReply: + LOGGER.info( + "Compute request: {:s}".format(str(grpc_message_to_json(request))) + ) + response = self.stub.Compute(request) + LOGGER.info( + "Compute result: {:s}".format(str(grpc_message_to_json(response))) + ) + return response + """ \ No newline at end of file diff --git a/src/vnt_manager/client/__init__.py b/src/vnt_manager/client/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/requirements.in b/src/vnt_manager/requirements.in new file mode 100644 index 000000000..4c4720a2d --- /dev/null +++ b/src/vnt_manager/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networkx \ No newline at end of file diff --git a/src/vnt_manager/service/VNTManagerService.py b/src/vnt_manager/service/VNTManagerService.py new file mode 100644 index 000000000..b61b213a6 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerService.py @@ -0,0 +1,35 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from common.Constants import ServiceNameEnum +from common.proto.vntmanager_pb2_grpc import add_VNTManagerServiceServicer_to_server +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from .VNTManagerServiceServicerImpl import VNTManagerServiceServicerImpl + +LOGGER = logging.getLogger(__name__) + + +class VNTManagerService(GenericGrpcService): + def __init__(self, cls_name: str = __name__): + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + super().__init__(port, cls_name=cls_name) + self.vntmanager_servicer = VNTManagerServiceServicerImpl() + + def install_servicers(self): + add_VNTManagerServiceServicer_to_server( + self.vntmanager_servicer, self.server + ) diff --git a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py new file mode 100644 index 000000000..4869218a7 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py @@ -0,0 +1,95 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import networkx as nx +import grpc +import copy + +from common.Constants import ServiceNameEnum +from common.method_wrappers.Decorator import (MetricsPool, MetricTypeEnum, safe_and_metered_rpc_method) +from common.proto.vntmanager_pb2 import VNTManagerRequest, VNTManagerReply +from common.proto.context_pb2 import Empty, Connection, EndPointId +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceServicer +from context.client.ContextClient import ContextClient +from context.service.database.uuids.EndPoint import endpoint_get_uuid + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool("VNTManager", "RPC") + +context_client: ContextClient = ContextClient() + + +class E2EOrchestratorServiceServicerImpl(VNTManagerServiceServicer): + def __init__(self): + LOGGER.debug("Creating Servicer...") + LOGGER.debug("Servicer Created") + + """ + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def Compute(self, request: E2EOrchestratorRequest, context: grpc.ServicerContext) -> E2EOrchestratorReply: + endpoints_ids = [] + for endpoint_id in request.service.service_endpoint_ids: + endpoints_ids.append(endpoint_get_uuid(endpoint_id)[2]) + + graph = nx.Graph() + + devices = context_client.ListDevices(Empty()).devices + + for device in devices: + endpoints_uuids = [endpoint.endpoint_id.endpoint_uuid.uuid + for endpoint in device.device_endpoints] + for ep in endpoints_uuids: + graph.add_node(ep) + + for ep in endpoints_uuids: + for ep_i in endpoints_uuids: + if ep == ep_i: + continue + graph.add_edge(ep, ep_i) + + links = context_client.ListLinks(Empty()).links + for link in links: + eps = [] + for endpoint_id in link.link_endpoint_ids: + eps.append(endpoint_id.endpoint_uuid.uuid) + graph.add_edge(eps[0], eps[1]) + + + shortest = nx.shortest_path(graph, endpoints_ids[0], endpoints_ids[1]) + + path = E2EOrchestratorReply() + path.services.append(copy.deepcopy(request.service)) + for i in range(0, int(len(shortest)/2)): + conn = Connection() + ep_a_uuid = str(shortest[i*2]) + ep_z_uuid = str(shortest[i*2+1]) + + conn.connection_id.connection_uuid.uuid = str(ep_a_uuid) + '_->_' + str(ep_z_uuid) + + ep_a_id = EndPointId() + ep_a_id.endpoint_uuid.uuid = ep_a_uuid + conn.path_hops_endpoint_ids.append(ep_a_id) + + ep_z_id = EndPointId() + ep_z_id.endpoint_uuid.uuid = ep_z_uuid + conn.path_hops_endpoint_ids.append(ep_z_id) + + path.connections.append(conn) + + return path + """ \ No newline at end of file diff --git a/src/vnt_manager/service/__init__.py b/src/vnt_manager/service/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/service/__main__.py b/src/vnt_manager/service/__main__.py new file mode 100644 index 000000000..03fb4dd5d --- /dev/null +++ b/src/vnt_manager/service/__main__.py @@ -0,0 +1,80 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import signal +import sys +import threading + +from prometheus_client import start_http_server + +from common.Constants import ServiceNameEnum +from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, + ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, + get_log_level, get_metrics_port, + wait_for_environment_variables) + +from .VNTManagerService import VNTManagerService + +terminate = threading.Event() +LOGGER = None + + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning("Terminate signal received") + terminate.set() + + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables( + [ + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_HOST), + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ] + ) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info("Starting...") + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting CentralizedCybersecurity service + grpc_service = VNTManagerService() + grpc_service.start() + LOGGER.info("Started...") + # Wait for Ctrl+C or termination signal + + while not terminate.wait(timeout=1): + pass + + + LOGGER.info("Terminating...") + grpc_service.stop() + + LOGGER.info("Bye") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab From 5ea83f3b9ddaa412a86830a93fc1fb62583bd29c Mon Sep 17 00:00:00 2001 From: sgambelluri Date: Wed, 27 Mar 2024 13:05:27 +0000 Subject: [PATCH 075/985] Endpoints Auto Descivory --- my_deploy.sh | 2 +- .../service/ContextServiceServicerImpl.py | 6 +- src/context/service/database/OpticalConfig.py | 74 ++++--- .../database/models/OpticalConfigModel.py | 42 +++- .../database/models/OpticalLinkModel.py | 78 +++++++ .../service/database/uuids/OpticalConfig.py | 17 ++ .../service/drivers/oc_driver/OCDriver.py | 5 +- .../drivers/oc_driver/templates/Tools.py | 11 +- src/tests/ofc24/run_test.sh | 17 ++ src/webui/service/__init__.py | 3 + src/webui/service/opticalconfig/__init__.py | 14 ++ src/webui/service/opticalconfig/forms.py | 22 ++ src/webui/service/opticalconfig/routes.py | 193 ++++++++++++++++++ src/webui/service/templates/base.html | 7 + .../opticalconfig/add_transceiver.html | 40 ++++ .../templates/opticalconfig/details.html | 109 ++++++++++ .../service/templates/opticalconfig/home.html | 68 ++++++ .../opticalconfig/update_interface.html | 73 +++++++ test.py | 41 ++++ 19 files changed, 783 insertions(+), 39 deletions(-) create mode 100644 src/context/service/database/models/OpticalLinkModel.py create mode 100644 src/context/service/database/uuids/OpticalConfig.py create mode 100644 src/tests/ofc24/run_test.sh create mode 100644 src/webui/service/opticalconfig/__init__.py create mode 100644 src/webui/service/opticalconfig/forms.py create mode 100644 src/webui/service/opticalconfig/routes.py create mode 100644 src/webui/service/templates/opticalconfig/add_transceiver.html create mode 100644 src/webui/service/templates/opticalconfig/details.html create mode 100644 src/webui/service/templates/opticalconfig/home.html create mode 100644 src/webui/service/templates/opticalconfig/update_interface.html create mode 100644 test.py diff --git a/my_deploy.sh b/my_deploy.sh index 7dd5e5c3e..9e4447349 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp opticalcontroller service slice nbi webui " # Uncomment to activate Monitoring #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index a102fa176..1be4cdac8 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -305,7 +305,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetOpticalConfig(self, request : Empty, context : grpc.ServicerContext) -> OpticalConfigList: result = get_opticalconfig(self.db_engine) - return OpticalConfigList(OpticalConfigs=result) + return OpticalConfigList(opticalconfigs=result) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SetOpticalConfig(self, request : OpticalConfig, context : grpc.ServicerContext) -> OpticalConfigId: @@ -316,5 +316,5 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer def SelectOpticalConfig(self, request : OpticalConfigId, context : grpc.ServicerContext) -> OpticalConfig: result = select_opticalconfig(self.db_engine, request) optical_config_id = OpticalConfigId() - optical_config_id.CopyFrom(result.OpticalConfig_id) - return OpticalConfig(config=result.config, OpticalConfig_id=optical_config_id) + optical_config_id.CopyFrom(result.opticalconfig_id) + return OpticalConfig(config=result.config, opticalconfig_id=optical_config_id) diff --git a/src/context/service/database/OpticalConfig.py b/src/context/service/database/OpticalConfig.py index 9e7552bc1..973e93c33 100644 --- a/src/context/service/database/OpticalConfig.py +++ b/src/context/service/database/OpticalConfig.py @@ -18,7 +18,8 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_cockroachdb import run_transaction from common.proto.context_pb2 import OpticalConfig, OpticalConfigId -from .models.OpticalConfigModel import OpticalConfigModel +from .models.OpticalConfigModel import OpticalConfigModel , OpticalChannelModel +from context.service.database.uuids.OpticalConfig import channel_get_uuid LOGGER = logging.getLogger(__name__) @@ -26,44 +27,61 @@ def get_opticalconfig(db_engine : Engine): def callback(session:Session): optical_configs = list() results = session.query(OpticalConfigModel).all() + for obj in results: + LOGGER.info(f"opticaln config obj from context {obj.dump()}") + optical_config = OpticalConfig() - optical_config.config = json.dump(obj.config) - optical_config.opticalconfig_id.opticalconfig_uuid = obj.opticalconfig_uuid + optical_config.config = json.dumps(obj.dump()) + optical_config.opticalconfig_id.opticalconfig_uuid = obj.dump_id()["opticalconfig_uuid"] optical_configs.append(optical_config) return optical_configs obj = run_transaction(sessionmaker(bind=db_engine), callback) return obj def set_opticalconfig(db_engine : Engine, request : OpticalConfig): + LOGGER.info(f"request {request} ") opticalconfig_id = OpticalConfigId() opticalconfig_id.opticalconfig_uuid = request.opticalconfig_id.opticalconfig_uuid - my_config_data = [] + OpticalConfig_data = [] if request.config: channels = [] transceivers = [] config = json.loads(request.config) - if 'channels' in config and len(config['channels']) > 0: - channels = [channel['name']['index'] for channel in config['channels']] + if 'transceivers' in config and len(config['transceivers']['transceiver']) > 0: transceivers = [transceiver for transceiver in config['transceivers']['transceiver']] - - my_config_data = [ - { - "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, - "channels" : channels, - "transcievers" : transceivers, - "interfaces" : json.dumps(config["interfaces"]["interface"]), - "channel_namespace" : config["channel_namespace"], - "endpoints" : [json.dumps(endpoint) for endpoint in config["endpoints"]], - "frequency" : config["frequency"] if "frequency" in config else 0, - "operational_mode" : config["operational_mode"] if "operational_mode" in config else 0, - "output_power" : config["output_power"] if "output_power" in config else '', - } - ] - + + if 'channels' in config and len(config['channels']) > 0: + #channels = [channel['name']['index'] for channel in config['channels']] + for channel_params in config['channels']: + channels.append( + { + "channel_uuid":channel_get_uuid(channel_params['name']['index']), + "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, + "channel_name" : channel_params['name']['index'], + "frequency" : int(channel_params["frequency"]) if "frequency" in channel_params else 0, + "operational_mode" : int(channel_params["operational-mode"]) if "operational-mode" in channel_params else 0, + "target_output_power" : channel_params["target-output-power"] if "target-output-power" in channel_params else '', + } + ) + + OpticalConfig_data.append( + { + "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, + "transcievers" : transceivers, + "interfaces" : json.dumps(config["interfaces"]["interface"]), + "channel_namespace" : config["channel_namespace"], + "endpoints" : [json.dumps(endpoint) for endpoint in config["endpoints"]],} + + ) + + + LOGGER.info(f"optical config to set {OpticalConfig_data} ") + LOGGER.info(f"channels {channels}") def callback(session:Session)->bool: - stmt = insert(OpticalConfigModel).values(my_config_data) + stmt = insert(OpticalConfigModel).values(OpticalConfig_data) + stmt = stmt.on_conflict_do_update( index_elements=[OpticalConfigModel.opticalconfig_uuid], set_=dict( @@ -71,7 +89,17 @@ def set_opticalconfig(db_engine : Engine, request : OpticalConfig): ) ) stmt = stmt.returning(OpticalConfigModel.opticalconfig_uuid) - id = session.execute(stmt).fetchone() + opticalconfig_id = session.execute(stmt).fetchone() + if (len(channels)>0) : + + stmt = insert(OpticalChannelModel).values(channels) + + stmt = stmt.on_conflict_do_nothing( + index_elements=[OpticalChannelModel.channel_uuid , OpticalConfigModel.opticalconfig_uuid], + + ) + stmt = stmt.returning(OpticalChannelModel.channel_uuid) + opticalChannel_id = session.execute(stmt).fetchone() opticalconfig_id = run_transaction(sessionmaker(bind=db_engine), callback) return {'opticalconfig_uuid': opticalconfig_id} diff --git a/src/context/service/database/models/OpticalConfigModel.py b/src/context/service/database/models/OpticalConfigModel.py index 10cf197f9..8531f37b2 100644 --- a/src/context/service/database/models/OpticalConfigModel.py +++ b/src/context/service/database/models/OpticalConfigModel.py @@ -13,30 +13,58 @@ # limitations under the License. import json -from sqlalchemy import Column, String, Integer +from sqlalchemy import Column, String, Integer , ForeignKey from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.orm import relationship from ._Base import _Base class OpticalConfigModel(_Base): __tablename__ = 'optical_config' opticalconfig_uuid = Column(String, primary_key=True) - channels = Column(ARRAY(String), nullable=True) + transcievers = Column(ARRAY(String), nullable=True) interfaces = Column(String, nullable=True) channel_namespace = Column(String, nullable=True) endpoints = Column(ARRAY(String), nullable=True) - frequency = Column(Integer, nullable=True) - operational_mode = Column(Integer, nullable=True) - output_power = Column(String, nullable=True) + channels = relationship("OpticalChannelModel") + + + def dump_id (self ): + return { + "opticalconfig_uuid":self.opticalconfig_uuid + } def dump(self): return { - "channels" : [{'name': {'index': channel}} for channel in self.channels], + "channels" : [channel.dump() for channel in self.channels], "transceivers" : {"transceiver": [transciever for transciever in self.transcievers]}, "interfaces" : {"interface": json.loads(self.interfaces)}, "channel_namespace" : self.channel_namespace, "endpoints" : [json.loads(endpoint) for endpoint in self.endpoints], + + } + + + +class OpticalChannelModel(_Base): + __tablename__ = 'optical_channel' + channel_uuid = Column(String, primary_key=True) + channel_name = Column (String,nullable=True) + frequency = Column(Integer, nullable=True) + operational_mode = Column(Integer, nullable=True) + target_output_power = Column(String, nullable=True) + opticalconfig_uuid = Column(ForeignKey('optical_config.opticalconfig_uuid', ondelete='CASCADE' ), primary_key=True) + opticalconfig = relationship('OpticalConfigModel', back_populates='channels') + def dump_id (self ): + return { + "channel_uuid":self.channel_uuid + } + + def dump(self): + return { + "name" :{'index':self.channel_name}, "frequency" : self.frequency, - "output_power" : self.output_power, + "target_output_power" : self.target_output_power, "operational_mode" : self.operational_mode, } + diff --git a/src/context/service/database/models/OpticalLinkModel.py b/src/context/service/database/models/OpticalLinkModel.py new file mode 100644 index 000000000..b94eeda93 --- /dev/null +++ b/src/context/service/database/models/OpticalLinkModel.py @@ -0,0 +1,78 @@ + +import operator +from sqlalchemy import CheckConstraint, Column, DateTime, Float, ForeignKey, Integer, String ,Boolean +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.types import ARRAY +from sqlalchemy.orm import relationship +from typing import Dict +from ._Base import _Base + +class OpticalLinkModel(_Base): + __tablename__ = 'opticallink' + + optical_link_uuid = Column(UUID(as_uuid=False), primary_key=True) + optical_link_name = Column(String, nullable=False) + length = Column(Integer, nullable=True) + source = Column(String, nullable=True) + target = Column(String, nullable=True) + optical_link_fiber= relationship("FiberModel") + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + + + + def dump_id(self) -> Dict: + return {'optical_link_uuid': {'uuid': self.link_uuid}} + + def dump(self) -> Dict: + result = { + 'optical_link_id' : self.dump_id(), + 'name' : self.optical_link_name, + 'details': { + "length" : self.length, + 'source' : self.source, + "target" : self.target, + 'fibers' : [ fiber.dump() for fiber in self.optical_link_fiber ] + } + + } + + return result + +class FiberModel(_Base): + __tablename__ = 'fiber' + fiber_uuid = Column(UUID(as_uuid=False), primary_key=True) + fiber_length = Column(Integer, nullable=True) + source_port = Column(String, nullable=True) + destination_port = Column(String, nullable=True) + local_peer_port = Column(String, nullable=True) + remote_peer_port = Column(String, nullable=True) + used = Column(Boolean ,nullable=true) + c_slots = Column (ARRAY(Integer),nullable=True) + l_slots = Column (ARRAY(Integer),nullable=True) + s_slots = Column (ARRAY(Integer),nullable=True) + optical_link_uuid = Column(ForeignKey('opticallink.optical_link_uuid', ondelete='CASCADE' ), primary_key=True) + optical_link = relationship('OpticalLinkModel', back_populates='optical_link_fibers') + + + def dump_id(self) -> Dict: + return {'fiber_uuid': {'uuid': self.fiber_uuid}} + + + def dump(self) -> Dict: + result = { + 'ID' : self.dump_id(), + 'length' : self.fiber_length, + "src_port" : self.source_port, + "dst_port" : self.destination_port, + "local_peer_port" : self.local_peer_port, + "remote_peer_port" : self.remote_peer_port, + "used" : self.used, + "c_slots" : self.c_slots , + "l_slots" : self.l_slots, + "s_slots" : self.s_slots + + } + + return result \ No newline at end of file diff --git a/src/context/service/database/uuids/OpticalConfig.py b/src/context/service/database/uuids/OpticalConfig.py new file mode 100644 index 000000000..0003b5712 --- /dev/null +++ b/src/context/service/database/uuids/OpticalConfig.py @@ -0,0 +1,17 @@ + +from common.method_wrappers.ServiceExceptions import InvalidArgumentsException +from ._Builder import get_uuid_from_string, get_uuid_random + +def channel_get_uuid( + channel_name :str , allow_random : bool = False +) -> str: + + + if len(channel_name) > 0: + return get_uuid_from_string(channel_name) + if allow_random: return get_uuid_random() + + raise InvalidArgumentsException([ + ('channel uuid', channel_name), + + ], extra_details=['Channel name is required to produce a channel UUID']) diff --git a/src/device/service/drivers/oc_driver/OCDriver.py b/src/device/service/drivers/oc_driver/OCDriver.py index 16f00cfb4..4b49c7e41 100644 --- a/src/device/service/drivers/oc_driver/OCDriver.py +++ b/src/device/service/drivers/oc_driver/OCDriver.py @@ -257,7 +257,8 @@ class OCDriver(_Driver): xml_data = self.__netconf_handler.get().data_xml transceivers,interfaces,channels_lst,channel_namespace,endpoints=extractor(data_xml=xml_data,resource_keys=filter_fields,dic=config) - + logging.info(f"xml response {xml_data}") + except Exception as e: # pylint: disable=broad-except MSG = 'Exception retrieving {:s}' self.__logger.info("error from getConfig %s",e) @@ -273,7 +274,7 @@ class OCDriver(_Driver): value_dic["interfaces"]=interfaces value_dic["channel_namespace"]=channel_namespace value_dic["endpoints"]=endpoints - + logging.info(f"parameters {value_dic}") opticalConfig.config=json.dumps(value_dic) opticalConfig.opticalconfig_id.opticalconfig_uuid=self.__device_uuid if self.__device_uuid is not None else "" config_id=context_client.SetOpticalConfig(opticalConfig) diff --git a/src/device/service/drivers/oc_driver/templates/Tools.py b/src/device/service/drivers/oc_driver/templates/Tools.py index 909bdd83b..bc69db6a6 100644 --- a/src/device/service/drivers/oc_driver/templates/Tools.py +++ b/src/device/service/drivers/oc_driver/templates/Tools.py @@ -140,9 +140,10 @@ def extract_channels_based_on_type (xml_data:str): return channel_names def extract_value(resource_key:str,xml_data:str,dic:dict,channel_name:str,channel_namespace:str): + logging.info(f"resource_key {resource_key} and channgel_name {channel_name} and channel_namespace {channel_namespace}") xml_bytes = xml_data.encode("utf-8") root = ET.fromstring(xml_bytes) - + channel_name=channel_name if 'index' not in channel_name else channel_name['index'] namespace = {'oc': 'http://openconfig.net/yang/platform', 'td': channel_namespace} @@ -153,10 +154,13 @@ def extract_value(resource_key:str,xml_data:str,dic:dict,channel_name:str,channe if (parameter is not None): value = parameter.text dic[resource_key]=value + else : + logging.info("parameter is None") else: + logging.info("element is None") print(" element not found.") - + logging.info(f"dic {dic}") return dic @@ -244,10 +248,11 @@ def extractor(data_xml:str,resource_keys:list,dic:dict): for channel_name in channel_names: dic={} - for resource_key in resource_keys : + for resource_key in resource_keys : if (resource_key != 'interface'): dic=extract_value(dic=dic,resource_key=resource_key,xml_data=data_xml,channel_name=channel_name,channel_namespace=channel_namespace) + dic["name"]=channel_name endpoints.append({"endpoint_uuid":{"uuid":channel_name}}) lst_dic.append(dic) diff --git a/src/tests/ofc24/run_test.sh b/src/tests/ofc24/run_test.sh new file mode 100644 index 000000000..660436552 --- /dev/null +++ b/src/tests/ofc24/run_test.sh @@ -0,0 +1,17 @@ +#!/bin/bash +docker stop na1 +docker rm na1 + screen -dmS t1 -T xterm sh -c "docker run -p 10.0.2.10:2023:2022 -v ~/tfs-ctrl/tempOC/files:/files --name na1 -it asgamb1/oc23bgp.img:latest +" + sleep 2 + if [ "$( docker container inspect -f '{{.State.Running}}' na1)" = "true" ]; then + + + + docker exec na1 sh -c " cp /files/platform_t1.xml demoECOC21.xml ; + + /confd/examples.confd/OC23/startNetconfAgent.sh; " + else + echo "your container is not running yet" + fi + diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index 63192016c..196652d3f 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -83,6 +83,9 @@ def create_app(use_config=None, web_app_root=None): from webui.service.load_gen.routes import load_gen # pylint: disable=import-outside-toplevel app.register_blueprint(load_gen) + + from webui.service.opticalconfig.routes import opticalconfig # pylint: disable=import-outside-toplevel + app.register_blueprint(opticalconfig) from webui.service.service.routes import service # pylint: disable=import-outside-toplevel app.register_blueprint(service) diff --git a/src/webui/service/opticalconfig/__init__.py b/src/webui/service/opticalconfig/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/webui/service/opticalconfig/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/webui/service/opticalconfig/forms.py b/src/webui/service/opticalconfig/forms.py new file mode 100644 index 000000000..30c133279 --- /dev/null +++ b/src/webui/service/opticalconfig/forms.py @@ -0,0 +1,22 @@ + + +from flask_wtf import FlaskForm +from wtforms import StringField, SelectField, TextAreaField, SubmitField +from wtforms.validators import DataRequired, Length, NumberRange, ValidationError +from common.proto.context_pb2 import DeviceOperationalStatusEnum + +class UpdateDeviceForm(FlaskForm): + power = StringField('Power') + frequency= StringField("Frequency") + operational_mode=StringField("Operational Mode") + line_port=SelectField("Line Port") + + + submit = SubmitField('Update') + +class AddTrancseiver (FlaskForm): + transceiver = StringField("Transceiver") + submit = SubmitField('Add') +class UpdateInterfaceForm (FlaskForm): + ip=StringField("IP Address") + prefix_length=StringField("Prefix Length") \ No newline at end of file diff --git a/src/webui/service/opticalconfig/routes.py b/src/webui/service/opticalconfig/routes.py new file mode 100644 index 000000000..5445d140e --- /dev/null +++ b/src/webui/service/opticalconfig/routes.py @@ -0,0 +1,193 @@ +import base64, json, logging #, re +from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request +from common.proto.context_pb2 import (ContextList, Empty, TopologyId, TopologyList +,DeviceId,DeviceList ,OpticalConfig, OpticalConfigId ,OpticalConfigList) +from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Topology import json_topology_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from service.client.ServiceClient import ServiceClient +from slice.client.SliceClient import SliceClient +from webui.service.main.forms import ContextTopologyForm, DescriptorForm +from .forms import UpdateDeviceForm ,AddTrancseiver ,UpdateInterfaceForm + + + +opticalconfig = Blueprint('opticalconfig', __name__,url_prefix="/opticalconfig") + +context_client = ContextClient() +device_client = DeviceClient() +service_client = ServiceClient() +slice_client = SliceClient() + +LOGGER = logging.getLogger(__name__) + +DESCRIPTOR_LOADER_NUM_WORKERS = 10 + +@opticalconfig.get("/") +def home() : + config=[] + deviceId= DeviceId() + + if 'context_uuid' not in session or 'topology_uuid' not in session: + flash("Please select a context!", "warning") + return redirect(url_for("main.home")) + context_uuid = session['context_uuid'] + topology_uuid = session['topology_uuid'] + + context_client.connect() + opticalConfig_list:OpticalConfigList = context_client.GetOpticalConfig(Empty()) + logging.info("myconfigList %s",opticalConfig_list) + + for configs in opticalConfig_list.opticalconfigs: + + value=json.loads(configs.config) if type(configs.config)==str else configs.config + value["channels_number"]=len(value['channels']) + + # value['operationalMode']=value['operational-mode'] + # value['targetOutputPower']=value['target-output-power'] + value['opticalconfig_id']=configs.opticalconfig_id + # value['line_port']=value["line-port"] + + config.append(value) + + logging.info("opticalConfig %s",config) + + context_client.close() + + return render_template( + 'opticalconfig/home.html', config=config) +@opticalconfig.route('/detail',methods=['GET']) +def show_details(config_uuid): + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + device_details=[] + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + LOGGER.info("response %s",response) + opticalConfig = OpticalConfig() + opticalConfig.CopyFrom(response) + + config =json.loads(opticalConfig.config) + LOGGER.info("config details %s",config) + interfaces=config["interfaces"] + new_config={} + + for channel in config['channels'] : + + new_config["name"]=channel['name'] + new_config['operationalMode']=channel['operational_mode'] if 'operational_mode' in channel else '' + new_config['targetOutputPower']=channel['target_output_power'] if 'target_output_power' in channel else '' + new_config["frequency"]=channel['frequency'] if 'frequency' in channel else '' + new_config['line_port']=channel["line-port"] if 'line-port' in channel else '' + + device_details.append(new_config) + LOGGER.info("config details %s",device_details) + return render_template('opticalconfig/details.html', device=device_details,config_id=config_uuid,interfaces=interfaces) +@opticalconfig.route('//update', methods=['GET', 'POST']) +def update(config_uuid,channel_name): + form = UpdateDeviceForm() + + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + LOGGER.info("response %s",response) + opticalconfig = OpticalConfig() + opticalconfig.CopyFrom(response) + config =json.loads(opticalconfig.config) + new_config={} + for channel in config['channels']: + if (channel["name"] == channel_name): + new_config=channel + form.frequency.default = channel["frequency"] + form.operational_mode.default=channel["operational-mode"] + form.power.default=channel["target-output-power"] + form.line_port.choices = [("","")] + + for transceiver in config["transceivers"]['transceiver']: + + form.line_port.choices.append((transceiver,transceiver)) + # listing enum values + + if form.validate_on_submit(): + + new_config["target-output-power"] =form.power.data if form.power.data != '' else new_config['target-output-power'] + new_config["frequency"]=form.frequency.data if form.frequency.data != '' else new_config['frequency'] + new_config["operational-mode"]=form.operational_mode.data if form.operational_mode.data != '' else new_config['operational-mode'] + new_config["line-port"]=form.line_port.data if form.line_port.data != '' else new_config['line-port'] + + opticalconfig.config =json.dumps(new_config) + LOGGER.info("myconfig copied %s",opticalconfig) + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticalconfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.show_details',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('myabout/update.html', device=response, form=form, submit_text='Update Device',channel_name=channel_name) +@opticalconfig.route('//update_interface', methods=['GET', 'POST']) +def update_interface (config_uuid,interface_name): + form = UpdateInterfaceForm() + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(myid) + context_client.close() + LOGGER.info("response %s",response) + opticalconfig = OpticalConfig() + opticalconfig.CopyFrom(response) + config =json.loads(opticalconfig.config) + new_config={} + if form.validate_on_submit(): + new_config["ip"]=form.ip.data if form.ip.data != "" else config["interfaces"]["interface"]["ip"] + new_config["prefix-length"]=form.prefix_length.data if form.prefix_length.data != "" else config["interfaces"]["interface"]["prefix-length"] + new_config["name"]=config["interfaces"]["interface"]["name"] + new_config["enabled"]=config["interfaces"]["interface"]["enabled"] + + opticalconfig.config=json.dumps({"update_interface":new_config}) + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticalconfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.show_details',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('opticalconfig/update_interface.html', + device=response, form=form, submit_text='Update interface',interface_name=interface_name) + +@opticalconfig.route('/add_transceiver', methods=['GET','POST']) +def add_transceiver (config_uuid): + config={} + addtrancseiver=AddTrancseiver() + opticalconfigId=OpticalConfig() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + opticlConfig=OpticalConfig() + opticlConfig.CopyFrom(response) + if addtrancseiver.validate_on_submit(): + config["add_transceiver"]=addtrancseiver.transceiver.data + opticlConfig.config=json.dumps(config) + + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticlConfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.update',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('opticalconfig/add_transceiver.html',form=addtrancseiver, submit_text='Add Trancseiver') + + + + \ No newline at end of file diff --git a/src/webui/service/templates/base.html b/src/webui/service/templates/base.html index 60cd5aebd..e081b2710 100644 --- a/src/webui/service/templates/base.html +++ b/src/webui/service/templates/base.html @@ -83,6 +83,13 @@ Slice {% endif %} +