diff --git a/src/device/requirements.in b/src/device/requirements.in index ece761571ec2ff9c3376b1062787d76047d71e7c..d8a33455e446b270ae5f1407c0d039fea889574f 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -15,6 +15,7 @@ anytree==2.8.0 APScheduler==3.10.1 +bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 Jinja2==3.0.3 @@ -32,7 +33,7 @@ tabulate ipaddress macaddress yattag -pyang +pyang==2.6.* git+https://github.com/robshakir/pyangbind.git websockets==10.4 diff --git a/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh new file mode 100755 index 0000000000000000000000000000000000000000..fe852f0e1520401b5edde1e61a726fbb6f03b225 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig + +rm -rf ${GIT_BASE_PATH} + +OC_PUBLIC_PATH=${GIT_BASE_PATH}/public +mkdir -p ${OC_PUBLIC_PATH} +git clone https://github.com/openconfig/public.git ${OC_PUBLIC_PATH} + +#OC_HERCULES_PATH=${GIT_BASE_PATH}/hercules +#mkdir -p ${OC_HERCULES_PATH} +#git clone https://github.com/openconfig/hercules.git ${OC_HERCULES_PATH} diff --git a/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh new file mode 100755 index 0000000000000000000000000000000000000000..ed4cf263f7e9fd29662c4b3fd4995338e691c362 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig +OC_PUBLIC_MODELS_PATH=${GIT_BASE_PATH}/public/release/models +IETF_MODELS_PATH=${GIT_BASE_PATH}/public/third_party/ietf +#OC_HERCULES_MODELS_PATH=${GIT_BASE_PATH}/hercules/yang + +OUT_FOLDER=openconfig +OUT_PATH=${BASE_PATH}/handlers +cd ${OUT_PATH} +export PYBINDPLUGIN=`/usr/bin/env python -c 'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'` + +# -p ${OC_HERCULES_MODELS_PATH}/ +# --split-class-dir openconfig_hercules +pyang --plugindir $PYBINDPLUGIN -p ${OC_PUBLIC_MODELS_PATH}/ -f pybind --split-class-dir ${OUT_FOLDER} \ + ${IETF_MODELS_PATH}/iana-if-type.yang \ + ${IETF_MODELS_PATH}/ietf-interfaces.yang \ + ${IETF_MODELS_PATH}/ietf-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv4-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv6-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match.yang \ + ${OC_PUBLIC_MODELS_PATH}/defined-sets/openconfig-defined-sets.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-aggregate.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ethernet.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ip.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/mpls/openconfig-mpls-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/openconfig-extensions.yang \ + ${OC_PUBLIC_MODELS_PATH}/optical-transport/openconfig-transport-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-common.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-controller-card.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-cpu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-ext.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fabric.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fan.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-healthz.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-integrated-circuit.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-linecard.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-pipeline-counters.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-port.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-psu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-software.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-transceiver.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-elements.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-mem-mgmt.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos.yang \ + ${OC_PUBLIC_MODELS_PATH}/system/openconfig-alarm-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-inet-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l2.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l3.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-errors.yang \ + + +openconfig-aft +openconfig-bgp +openconfig-igmp +openconfig-isis +openconfig-local-routing +openconfig-mpls +openconfig-ospfv2 +openconfig-pcep +openconfig-pim +openconfig-policy-forwarding +openconfig-policy-types +openconfig-routing-policy +openconfig-segment-routing + + + + +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-interfaces.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-chassis.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-linecard.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-node.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-port.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-qos.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules.yang \ diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 04dae4f5fcc6427c735b528b0ab32ba1c967709a..6f80ee82faf3cb7d094a92951d79cae0cd44669e 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -132,8 +132,8 @@ class GnmiSessionHandler: #resource_key_tuple[2] = True results.extend(parse(str_path, value)) except Exception as e: # pylint: disable=broad-except - MSG = 'Exception processing notification {:s}' - self._logger.exception(MSG.format(grpc_message_to_json_string(notification))) + MSG = 'Exception processing update {:s}' + self._logger.exception(MSG.format(grpc_message_to_json_string(update))) results.append((str_path, e)) # if validation fails, store the exception #_results = sorted(results.items(), key=lambda x: x[1][0]) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 0b3c1f9705353548025cb4365ea31e68978c79f1..cddf40d56454582942485ab49b00744b70c2d69f 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,45 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging #, json +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +#pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' - def get_path(self) -> str: return '/components/component' + def get_path(self) -> str: return '/openconfig-platform:components' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_component_list : List[Dict] = json_data.get('component', []) - response = [] - for json_component in json_component_list: - #LOGGER.info('json_component = {:s}'.format(json.dumps(json_component))) - endpoint = {} + oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') + #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) - component_type = json_component.get('state', {}).get('type') - if component_type is None: continue - component_type = component_type.replace('oc-platform-types:', '') - component_type = component_type.replace('openconfig-platform-types:', '') - if component_type not in {'PORT'}: continue - endpoint['type'] = '-' + entries = [] + for component_key, oc_component in oc_components.component.items(): + #LOGGER.info('component_key={:s} oc_component={:s}'.format( + # component_key, pybindJSON.dumps(oc_component, mode='ietf') + #)) - #LOGGER.info('PORT json_component = {:s}'.format(json.dumps(json_component))) + component_name = oc_component.config.name - component_name = json_component.get('name') - if component_name is None: continue + component_type = oc_component.state.type + component_type = component_type.split(':')[-1] + if component_type not in {'PORT'}: continue # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. interface_name = component_name.lower().replace('-port', '') - endpoint['uuid'] = interface_name + endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED : PATH_IF_CTR.format(interface_name, 'in-octets' ), KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED : PATH_IF_CTR.format(interface_name, 'out-octets'), @@ -59,5 +59,7 @@ class ComponentHandler(_Handler): } if len(endpoint) == 0: continue - response.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - return response + + entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 20f79b3c2e15b58ab99166a68422fd35d40fd00f..77310d51d544339f10858445b0260f645cfdee60 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -13,15 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' - def get_path(self) -> str: return '/interfaces/interface' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: if_name = str (resource_value['name' ]) # ethernet-1/1 @@ -63,186 +64,88 @@ class InterfaceHandler(_Handler): def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - #LOGGER.info('json_interface = {:s}'.format(json.dumps(json_interface))) + entries = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + #)) interface = {} + interface['name'] = oc_interface.config.name - interface_name = json_interface.get('name') - if interface_name is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue - interface['name'] = interface_name - - CONFIG_FIELDS = ('config', 'openconfig-interface:config', 'oci:config') - json_config : Dict = dict_get_first(json_interface, CONFIG_FIELDS, default={}) - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state : Dict = dict_get_first(json_interface, STATE_FIELDS, default={}) - - interface_type = json_config.get('type') - if interface_type is None: interface_type = json_state.get('type') - if interface_type is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue + interface_type = oc_interface.config.type interface_type = interface_type.replace('ianaift:', '') interface_type = interface_type.replace('iana-if-type:', '') interface['type'] = interface_type - interface_mtu = json_config.get('mtu') - if interface_mtu is None: interface_mtu = json_state.get('mtu') - if interface_mtu is not None: interface['mtu'] = int(interface_mtu) - - interface_enabled = json_config.get('enabled') - if interface_enabled is None: interface_enabled = json_state.get('enabled') - interface['enabled'] = False if interface_enabled is None else bool(interface_enabled) - - interface_management = json_config.get('management') - if interface_management is None: interface_management = json_state.get('management') - interface['management'] = False if interface_management is None else bool(interface_management) + interface['mtu' ] = oc_interface.config.mtu + interface['enabled' ] = oc_interface.config.enabled + interface['description' ] = oc_interface.config.description + interface['admin-status'] = oc_interface.state.admin_status + interface['oper-status' ] = oc_interface.state.oper_status + interface['management' ] = oc_interface.state.management - interface_descr = json_interface.get('config', {}).get('description') - if interface_descr is not None: interface['description'] = interface_descr + entry_interface_key = '/interface[{:s}]'.format(interface['name']) + entries.append((entry_interface_key, interface)) - json_subinterfaces = json_interface.get('subinterfaces', {}) - json_subinterface_list : List[Dict] = json_subinterfaces.get('subinterface', []) - - for json_subinterface in json_subinterface_list: - #LOGGER.info('json_subinterface = {:s}'.format(json.dumps(json_subinterface))) + for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): + #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( + # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') + #)) subinterface = {} + subinterface['index' ] = oc_subinterface.state.index + subinterface['name' ] = oc_subinterface.state.name + subinterface['enabled'] = oc_subinterface.state.enabled - subinterface_index = json_subinterface.get('state', {}).get('index') - if subinterface_index is None: continue - subinterface['index'] = int(subinterface_index) - - subinterface_name = json_subinterface.get('state', {}).get('name') - if subinterface_name is None: continue - subinterface['name'] = subinterface_name - - subinterface_enabled = json_subinterface.get('state', {}).get('enabled', False) - subinterface['enabled'] = bool(subinterface_enabled) - - VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - - # TODO: implement support for multiple IP addresses per subinterface - - IPV4_FIELDS = ('ipv4', 'openconfig-if-ip:ipv4', 'ociip:ipv4') - json_ipv4 = dict_get_first(json_subinterface, IPV4_FIELDS, default={}) - - IPV4_ADDRESSES_FIELDS = ('addresses', 'openconfig-if-ip:addresses', 'ociip:addresses') - json_ipv4_addresses = dict_get_first(json_ipv4, IPV4_ADDRESSES_FIELDS, default={}) - - IPV4_ADDRESS_FIELDS = ('address', 'openconfig-if-ip:address', 'ociip:address') - json_ipv4_address_list : List[Dict] = dict_get_first(json_ipv4_addresses, IPV4_ADDRESS_FIELDS, default=[]) - - #ipv4_addresses = [] - for json_ipv4_address in json_ipv4_address_list: - #LOGGER.info('json_ipv4_address = {:s}'.format(json.dumps(json_ipv4_address))) + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) + entries.append((entry_subinterface_key, subinterface)) - STATE_FIELDS = ('state', 'openconfig-if-ip:state', 'ociip:state') - json_ipv4_address_state = dict_get_first(json_ipv4_address, STATE_FIELDS, default={}) + #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') + #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - #ipv4_address = {} + #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') + #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - #ORIGIN_FIELDS = ('origin', 'openconfig-if-ip:origin', 'ociip:origin') - #ipv4_address_origin = dict_get_first(json_ipv4_address_state, ORIGIN_FIELDS, default={}) - #if ipv4_address_origin is not None: ipv4_address['origin'] = ipv4_address_origin + #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') + #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - IP_FIELDS = ('ip', 'openconfig-if-ip:ip', 'ociip:ip') - ipv4_address_ip = dict_get_first(json_ipv4_address_state, IP_FIELDS) - #if ipv4_address_ip is not None: ipv4_address['address_ip'] = ipv4_address_ip - if ipv4_address_ip is not None: subinterface['address_ip'] = ipv4_address_ip + #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') + #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - PREFIX_FIELDS = ('prefix-length', 'openconfig-if-ip:prefix-length', 'ociip:prefix-length') - ipv4_address_prefix = dict_get_first(json_ipv4_address_state, PREFIX_FIELDS) - #if ipv4_address_prefix is not None: ipv4_address['address_prefix'] = int(ipv4_address_prefix) - if ipv4_address_prefix is not None: subinterface['address_prefix'] = int(ipv4_address_prefix) + #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') + #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) + #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - #if len(ipv4_address) == 0: continue - #ipv4_addresses.append(ipv4_address) + for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): + #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #subinterface['ipv4_addresses'] = ipv4_addresses - - if len(subinterface) == 0: continue - resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) - response.append((resource_key, subinterface)) - - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) - - return response - - def parse_counters(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse_counters] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) - - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse_counters] json_interface = {:s}'.format(json.dumps(json_interface))) - - interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) + address_ipv4 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) + entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) + entries.append((entry_address_ipv4_key, address_ipv4)) - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) + for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): + #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #LOGGER.info('[parse_counters] interface = {:s}'.format(str(interface))) + address_ipv6 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) + entries.append((entry_address_ipv6_key, address_ipv6)) - return response + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index a45dc9e7f972445691143df15d6d56d079384fc4..1c2cfc17aea4498cbcac7c3460cc53862847cf70 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -14,67 +14,51 @@ import json, logging from typing import Any, Dict, List, Tuple +import pyangbind.lib.pybindJSON as pybindJSON +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) +#pylint: disable=abstract-method class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' - def get_path(self) -> str: return '/interfaces/interface/state/counters' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse] json_interface = {:s}'.format(json.dumps(json_interface))) + counters = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + )) interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) - - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) - - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) - - #LOGGER.info('[parse] interface = {:s}'.format(str(interface))) + interface['name'] = oc_interface.name + + interface_counters = oc_interface.state.counters + interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts + interface['in-discards' ] = interface_counters.in_discards + interface['in-errors' ] = interface_counters.in_errors + interface['in-fcs-errors' ] = interface_counters.in_fcs_errors + interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts + interface['in-octets' ] = interface_counters.in_octets + interface['in-pkts' ] = interface_counters.in_pkts + interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts + interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts + interface['out-discards' ] = interface_counters.out_discards + interface['out-errors' ] = interface_counters.out_errors + interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts + interface['out-octets' ] = interface_counters.out_octets + interface['out-pkts' ] = interface_counters.out_pkts + interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + + LOGGER.info('interface = {:s}'.format(str(interface))) if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + counters.append(('/interface[{:s}]'.format(interface['name']), interface)) - return response + return counters diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index aed821a06fa7fcafe96a21ad5f5fa06be2902038..c29ed263a81246b9084c389664e6064d9bc12772 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -13,14 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' - def get_path(self) -> str: return '/network-instances/network-instance' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: ni_name = str(resource_value['name']) # test-svc @@ -58,5 +60,10 @@ class NetworkInstanceHandler(_Handler): } def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) response = [] return response + +openconfig-network-instance:network-instance \ No newline at end of file diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 30343ac28a46a0c1d24bcb66d07fa03fa377f9fa..8cf704e2980e94e86e6d8445b8ad71a863434fe2 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -23,8 +23,20 @@ def get_schema(resource_key : str): resource_key = RE_REMOVE_NAMESPACES.sub('/', resource_key) return resource_key -def dict_get_first(d : Dict, field_names : Iterable[str], default=None) -> Any: - for field_name in field_names: - if field_name not in d: continue - return d[field_name] +def container_get_first( + container : Dict[str, Any], key_name : str, namespace : Optional[str]=None, namespaces : Iterable[str]=tuple(), + default : Optional[Any] = None +) -> Any: + value = container.get(key_name) + if value is not None: return value + + if namespace is not None: + if len(namespaces) > 0: + raise Exception('At maximum, one of namespace or namespaces can be specified') + namespaces = (namespace,) + + for namespace in namespaces: + namespace_key_name = '{:s}:{:s}'.format(namespace, key_name) + if namespace_key_name in container: return container[namespace_key_name] + return default diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 39cd7c66ad5e8c16e89192ad0f2ffb7c43ae6c50..6d54ef28d4bf1af1697e0fccd2bb1b6253366422 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -46,9 +46,10 @@ RESOURCE_KEY_MAPPER = { } PATH_MAPPER = { - '/components' : comph.get_path(), - '/interfaces' : ifaceh.get_path(), - '/network-instances' : nih.get_path(), + '/components' : comph.get_path(), + '/components/component' : comph.get_path(), + '/interfaces' : ifaceh.get_path(), + '/network-instances' : nih.get_path(), } RESOURCE_KEY_TO_HANDLER = { @@ -88,9 +89,9 @@ def get_handler( path_schema = PATH_MAPPER.get(path_schema, path_schema) handler = PATH_TO_HANDLER.get(path_schema) if handler is None and raise_if_not_found: - MSG = 'Handler not found: resource_key={:s} resource_key_schema={:s}' + MSG = 'Handler not found: path={:s} path_schema={:s}' # pylint: disable=broad-exception-raised - raise Exception(MSG.format(str(resource_key), str(resource_key_schema))) + raise Exception(MSG.format(str(path), str(path_schema))) return handler def get_path(resource_key : str) -> str: diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Path.py b/src/device/service/drivers/gnmi_openconfig/tools/Path.py index 40ab28dc6bbaf8a65b667804dfe9285f36864e29..2d6dc1e74f201f262be50d7d07217a6ce04f4c9b 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Path.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Path.py @@ -19,8 +19,8 @@ from ..gnmi.gnmi_pb2 import Path, PathElem RE_PATH_SPLIT = re.compile(r'/(?=(?:[^\[\]]|\[[^\[\]]+\])*$)') RE_PATH_KEYS = re.compile(r'\[(.*?)\]') -def path_from_string(path='/'): - if not path: return Path(elem=[]) +def path_from_string(path='/'): #, origin='openconfig' + if not path: return Path(elem=[]) #, origin=origin if path[0] == '/': if path[-1] == '/': @@ -40,7 +40,7 @@ def path_from_string(path='/'): dict_keys = dict(x.split('=', 1) for x in elem_keys) path.append(PathElem(name=elem_name, key=dict_keys)) - return Path(elem=path) + return Path(elem=path) #, origin=origin def path_to_string(path : Path) -> str: path_parts = list() diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 4797930a17360d8a780e99ea9ac05c0e3a1f7abc..9933cb8584e657608dd9e2010021597a6083605a 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -13,9 +13,36 @@ # limitations under the License. import base64, json -from typing import Any +from typing import Any, Dict, List, Union from ..gnmi.gnmi_pb2 import TypedValue +REMOVE_NAMESPACES = ( + 'arista-intf-augments', + 'arista-netinst-augments', + 'openconfig-hercules-platform', +) + +def remove_fields(key : str) -> bool: + parts = key.split(':') + if len(parts) == 1: return False + namespace = parts[0].lower() + return namespace in REMOVE_NAMESPACES + +def recursive_remove_keys(container : Union[Dict, List, Any]) -> None: + if isinstance(container, dict): + remove_keys = [ + key + for key in container.keys() + if remove_fields(key) + ] + for key in remove_keys: + container.pop(key, None) + for value in container.values(): + recursive_remove_keys(value) + elif isinstance(container, list): + for value in container: + recursive_remove_keys(value) + def decode_value(value : TypedValue) -> Any: encoding = value.WhichOneof('value') if encoding == 'json_val': @@ -31,9 +58,13 @@ def decode_value(value : TypedValue) -> Any: raise NotImplementedError() #return value elif encoding == 'json_ietf_val': - value : str = value.json_ietf_val + str_value : str = value.json_ietf_val.decode('UTF-8') try: - return json.loads(value) + # Cleanup and normalize the records according to OpenConfig + str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + json_value = json.loads(str_value) + recursive_remove_keys(json_value) + return json_value except json.decoder.JSONDecodeError: # Assume is Base64-encoded b_b64_value = value.encode('UTF-8') diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py index 50c9155822d5285fda5fc75777363c066ffb215a..684b9f4c3b752dc099f276d2d60f9549e5cede19 100644 --- a/src/device/tests/test_gnmi.py +++ b/src/device/tests/test_gnmi.py @@ -16,9 +16,9 @@ import logging, os, sys, time from typing import Dict, Tuple os.environ['DEVICE_EMULATED_ONLY'] = 'YES' from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -#from device.service.driver_api._Driver import ( -# RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -#) +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -58,21 +58,21 @@ def main(): driver_settings = { 'protocol': 'gnmi', 'username': 'admin', - 'password': 'NokiaSrl1!', - 'use_tls' : True, + 'password': 'admin', + 'use_tls' : False, } - driver = GnmiOpenConfigDriver('172.100.100.102', 57400, **driver_settings) + driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) driver.Connect() #resources_to_get = [] #resources_to_get = [RESOURCE_ENDPOINTS] #resources_to_get = [RESOURCE_INTERFACES] - #resources_to_get = [RESOURCE_NETWORK_INSTANCES] + resources_to_get = [RESOURCE_NETWORK_INSTANCES] #resources_to_get = [RESOURCE_ROUTING_POLICIES] #resources_to_get = [RESOURCE_SERVICES] - #LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - #results_getconfig = driver.GetConfig(resources_to_get) - #LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) #resources_to_set = [ # network_instance('test-svc', 'L3VRF'), @@ -90,21 +90,21 @@ def main(): #results_setconfig = driver.SetConfig(resources_to_set) #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - resources_to_delete = [ - #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - - #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - - interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - - network_instance('20f66fb5', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + #resources_to_delete = [ + # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), + # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), + # + # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), + # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), + # + # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), + # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), + # + # #network_instance('20f66fb5', 'L3VRF'), + #] + #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + #results_deleteconfig = driver.DeleteConfig(resources_to_delete) + #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) time.sleep(1) diff --git a/test_gnmi.sh b/test_gnmi.sh new file mode 100755 index 0000000000000000000000000000000000000000..d1fe36969cdb3a0778d8db456bba7458da995b48 --- /dev/null +++ b/test_gnmi.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export PYTHONPATH=./src +python -m device.tests.test_gnmi