Commit 48c2413c authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Device component - gNMI/OpenConfig Driver: 

WORK IN PROGRESS
- Added unitary tests and scripts
- Enhanced reporting of capabilities
- Migrated Component and Interface code to libyang
- Migrating NetworkInstance code to libyang
- Disabled unneeded log messages
- Temporarily disabled telemetry
- Added LibYang-based YANG handler
- Added helper methods
parent 22082f10

scripts/run_tests_locally-device-gnmi-openconfig.sh

0 → 100755
+25 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





PROJECTDIR=`pwd`



cd $PROJECTDIR/src

RCFILE=$PROJECTDIR/coverage/.coveragerc



# Run unitary tests and analyze coverage of code at same time

# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    device/tests/test_unitary_gnmi_openconfig.py

src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py

+37 −28
Original line number Diff line number Diff line
@@ -19,12 +19,13 @@ from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk 
from .gnmi.gnmi_pb2_grpc import gNMIStub

from .gnmi.gnmi_pb2 import Encoding, GetRequest, SetRequest, UpdateResult   # pylint: disable=no-name-in-module

from .handlers import ALL_RESOURCE_KEYS, compose, get_path, parse

from .tools.Capabilities import get_supported_encodings

from .handlers.YangHandler import YangHandler

from .tools.Capabilities import check_capabilities

from .tools.Channel import get_grpc_channel

from .tools.Path import path_from_string, path_to_string #, compose_path

from .tools.Subscriptions import Subscriptions

from .tools.Value import decode_value #, value_exists

from .MonitoringThread import MonitoringThread

#from .MonitoringThread import MonitoringThread



class GnmiSessionHandler:

    def __init__(self, address : str, port : int, settings : Dict, logger : logging.Logger) -> None:
@@ -39,12 +40,20 @@ class GnmiSessionHandler: 
        self._use_tls   = settings.get('use_tls', False)

        self._channel : Optional[grpc.Channel] = None

        self._stub : Optional[gNMIStub] = None

        self._monit_thread = None

        self._supported_encodings = None

        self._yang_handler = YangHandler()

        #self._monit_thread = None

        self._subscriptions = Subscriptions()

        self._in_subscriptions = queue.Queue()

        self._out_samples = queue.Queue()



    def __del__(self) -> None:

        self._logger.warning('Destroying YangValidator...')

        self._logger.warning('yang_validator.data:')

        for path, dnode in self._yang_handler.get_data_paths().items():

            self._logger.warning('  {:s}: {:s}'.format(str(path), json.dumps(dnode.print_dict())))

        self._yang_handler.destroy()

        self._logger.warning('DONE')



    @property

    def subscriptions(self): return self._subscriptions
@@ -58,18 +67,17 @@ class GnmiSessionHandler: 
        with self._lock:

            self._channel = get_grpc_channel(self._address, self._port, self._use_tls, self._logger)

            self._stub = gNMIStub(self._channel)

            self._supported_encodings = get_supported_encodings(

                self._stub, self._username, self._password, timeout=120)

            self._monit_thread = MonitoringThread(

                self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples)

            self._monit_thread.start()

            check_capabilities(self._stub, self._username, self._password, timeout=120)

            #self._monit_thread = MonitoringThread(

            #    self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples)

            #self._monit_thread.start()

            self._connected.set()



    def disconnect(self):

        if not self._connected.is_set(): return

        with self._lock:

            self._monit_thread.stop()

            self._monit_thread.join()

            #self._monit_thread.stop()

            #self._monit_thread.join()

            self._channel.close()

            self._connected.clear()
@@ -87,9 +95,9 @@ class GnmiSessionHandler: 
            str_resource_name = 'resource_key[#{:d}]'.format(i)

            try:

                chk_string(str_resource_name, resource_key, allow_empty=False)

                self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key)))

                #self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key)))

                str_path = get_path(resource_key)

                self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path)))

                #self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path)))

                get_request.path.append(path_from_string(str_path))

            except Exception as e: # pylint: disable=broad-except

                MSG = 'Exception parsing {:s}: {:s}'
@@ -130,7 +138,7 @@ class GnmiSessionHandler: 
                    value = decode_value(update.val)

                    #resource_key_tuple[1] = value

                    #resource_key_tuple[2] = True

                    results.extend(parse(str_path, value))

                    results.extend(parse(str_path, value, self._yang_handler))

                except Exception as e: # pylint: disable=broad-except

                    MSG = 'Exception processing update {:s}'

                    self._logger.exception(MSG.format(grpc_message_to_json_string(update)))
@@ -159,17 +167,17 @@ class GnmiSessionHandler: 
        set_request = SetRequest()

        #for resource_key in resource_keys:

        for resource_key, resource_value in resources:

            self._logger.info('---1')

            self._logger.info(str(resource_key))

            self._logger.info(str(resource_value))

            #self._logger.info('---1')

            #self._logger.info(str(resource_key))

            #self._logger.info(str(resource_value))

            #resource_tuple = resource_tuples.get(resource_key)

            #if resource_tuple is None: continue

            #_, value, exists, operation_done = resource_tuple

            if isinstance(resource_value, str): resource_value = json.loads(resource_value)

            str_path, str_data = compose(resource_key, resource_value, delete=False)

            self._logger.info('---3')

            self._logger.info(str(str_path))

            self._logger.info(str(str_data))

            str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=False)

            #self._logger.info('---3')

            #self._logger.info(str(str_path))

            #self._logger.info(str(str_data))

            set_request_list = set_request.update #if exists else set_request.replace

            set_request_entry = set_request_list.add()

            set_request_entry.path.CopyFrom(path_from_string(str_path))
@@ -228,18 +236,19 @@ class GnmiSessionHandler: 
        set_request = SetRequest()

        #for resource_key in resource_keys:

        for resource_key, resource_value in resources:

            self._logger.info('---1')

            self._logger.info(str(resource_key))

            self._logger.info(str(resource_value))

            #self._logger.info('---1')

            #self._logger.info(str(resource_key))

            #self._logger.info(str(resource_value))

            #resource_tuple = resource_tuples.get(resource_key)

            #if resource_tuple is None: continue

            #_, value, exists, operation_done = resource_tuple

            #if not exists: continue

            if isinstance(resource_value, str): resource_value = json.loads(resource_value)

            str_path, str_data = compose(resource_key, resource_value, delete=True)

            self._logger.info('---3')

            self._logger.info(str(str_path))

            self._logger.info(str(str_data))

            # pylint: disable=unused-variable

            str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=True)

            #self._logger.info('---3')

            #self._logger.info(str(str_path))

            #self._logger.info(str(str_data))

            set_request_entry = set_request.delete.add()

            set_request_entry.CopyFrom(path_from_string(str_path))

src/device/service/drivers/gnmi_openconfig/handlers/Component.py

+22 −17
Original line number Diff line number Diff line
@@ -12,37 +12,44 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import logging #, json

import pyangbind.lib.pybindJSON as pybindJSON

import json, logging # libyang

from typing import Any, Dict, List, Tuple

from common.proto.kpi_sample_types_pb2 import KpiSampleType

from . import openconfig

from ._Handler import _Handler

from .YangHandler import YangHandler



LOGGER = logging.getLogger(__name__)



PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}"

PATH_IF_CTR = '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}'



#pylint: disable=abstract-method

class ComponentHandler(_Handler):

    def get_resource_key(self) -> str: return '/endpoints/endpoint'

    def get_path(self) -> str: return '/openconfig-platform:components'



    def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]:

        #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data)))

    def parse(

        self, json_data : Dict, yang_handler : YangHandler

    ) -> List[Tuple[str, Dict[str, Any]]]:

        LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data)))



        oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components')

        #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf')))

        yang_components_path = self.get_path()

        json_data_valid = yang_handler.parse_to_dict(yang_components_path, json_data, fmt='json')



        entries = []

        for component_key, oc_component in oc_components.component.items():

            #LOGGER.info('component_key={:s} oc_component={:s}'.format(

            #    component_key, pybindJSON.dumps(oc_component, mode='ietf')

            #))

        for component in json_data_valid['components']['component']:

            LOGGER.debug('component={:s}'.format(str(component)))



            component_name = oc_component.config.name

            component_name = component['name']

            #component_config = component.get('config', {})



            component_type = oc_component.state.type

            #yang_components : libyang.DContainer = yang_handler.get_data_path(yang_components_path)

            #yang_component_path = 'component[name="{:s}"]'.format(component_name)

            #yang_component : libyang.DContainer = yang_components.create_path(yang_component_path)

            #yang_component.merge_data_dict(component, strict=True, validate=False)



            component_state = component.get('state', {})

            component_type = component_state.get('type')

            if component_type is None: continue

            component_type = component_type.split(':')[-1]

            if component_type not in {'PORT'}: continue
@@ -58,8 +65,6 @@ class ComponentHandler(_Handler): 
                KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED: PATH_IF_CTR.format(interface_name, 'out-pkts'  ),

            }



            if len(endpoint) == 0: continue



            entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint))



        return entries

src/device/service/drivers/gnmi_openconfig/handlers/Interface.py

+163 −113

File changed.

Preview size limit exceeded, changes collapsed.

src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py

+117 −28
Original line number Diff line number Diff line
@@ -12,20 +12,40 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import json, logging

import pyangbind.lib.pybindJSON as pybindJSON

import json, libyang, logging

import operator

from typing import Any, Dict, List, Tuple

from . import openconfig

from ._Handler import _Handler

from .Tools import get_bool, get_int, get_str

from .YangHandler import YangHandler



LOGGER = logging.getLogger(__name__)



MAP_NETWORK_INSTANCE_TYPE = {

    # special routing instance; acts as default/global routing instance for a network device

    'DEFAULT': 'openconfig-network-instance-types:DEFAULT_INSTANCE',



    # private L3-only routing instance; formed of one or more RIBs

    'L3VRF': 'openconfig-network-instance-types:L3VRF',



    # private L2-only switch instance; formed of one or more L2 forwarding tables

    'L2VSI': 'openconfig-network-instance-types:L2VSI',



    # private L2-only forwarding instance; point to point connection between two endpoints

    'L2P2P': 'openconfig-network-instance-types:L2P2P',



    # private Layer 2 and Layer 3 forwarding instance

    'L2L3': 'openconfig-network-instance-types:L2L3',

}



class NetworkInstanceHandler(_Handler):

    def get_resource_key(self) -> str: return '/network_instance'

    def get_path(self) -> str: return '/openconfig-network-instance:network-instances'



    def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]:

        ni_name   = str(resource_value['name'])   # test-svc

    def compose(

        self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False

    ) -> Tuple[str, str]:

        ni_name   = get_str(resource_value, 'name') # test-svc



        if delete:

            PATH_TMPL = '/network-instances/network-instance[name={:s}]'
@@ -33,15 +53,11 @@ class NetworkInstanceHandler(_Handler): 
            str_data = json.dumps({})

            return str_path, str_data



        ni_type   = str(resource_value['type'])   # L3VRF / L2VSI / ...

        ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ...

        ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type)



        # not works: [FailedPrecondition] unsupported identifier 'DIRECTLY_CONNECTED'

        #protocols = [self._compose_directly_connected()]

        # 'DIRECTLY_CONNECTED' is implicitly added



        MAP_OC_NI_TYPE = {

            'L3VRF': 'openconfig-network-instance-types:L3VRF',

        }

        ni_type = MAP_OC_NI_TYPE.get(ni_type, ni_type)



        str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name)

        str_data = json.dumps({
@@ -51,19 +67,92 @@ class NetworkInstanceHandler(_Handler): 
        })

        return str_path, str_data



    def _compose_directly_connected(self, name=None, enabled=True) -> Dict:

        identifier = 'DIRECTLY_CONNECTED'

        if name is None: name = 'DIRECTLY_CONNECTED'

        return {

            'identifier': identifier, 'name': name,

            'config': {'identifier': identifier, 'name': name, 'enabled': enabled},

    def parse(

        self, json_data : Dict, yang_handler : YangHandler

    ) -> List[Tuple[str, Dict[str, Any]]]:

        LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data)))



        # Arista Parsing Fixes:

        # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0

        #   overwrite with .../hellos/config/hello-interval

        network_instances = json_data.get('openconfig-network-instance:network-instance', [])

        for network_instance in network_instances:

            if network_instance['name'] != 'default': continue

            mpls_rsvp_te = network_instance.get('mpls', {}).get('signaling-protocols', {}).get('rsvp-te', {})

            mpls_rsvp_te_hellos = mpls_rsvp_te.get('global', {}).get('hellos', {})

            hello_interval = mpls_rsvp_te_hellos.get('config', {}).get('hello-interval', 9000)

            mpls_rsvp_te_hellos.get('state', {})['hello-interval'] = hello_interval



        yang_network_instances_path = self.get_path()

        json_data_valid = yang_handler.parse_to_dict(yang_network_instances_path, json_data, fmt='json', strict=False)



        entries = []

        for network_instance in json_data_valid['network-instances']['network-instance']:

            LOGGER.debug('network_instance={:s}'.format(str(network_instance)))

            ni_name = network_instance['name']



            ni_config = network_instance['config']

            ni_type = ni_config['type'].split(':')[-1]



            _net_inst = {'name': ni_name, 'type': ni_type}

            entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name)

            entries.append((entry_net_inst_key, _net_inst))



            ni_protocols = network_instance.get('protocols', {}).get('protocol', [])

            for ni_protocol in ni_protocols:

                ni_protocol_id = ni_protocol['identifier'].split(':')[-1]

                ni_protocol_name = ni_protocol['name']



                _protocol = {'id': ni_protocol_id, 'name': ni_protocol_name}

                entry_protocol_key = '{:s}/protocol[{:s}]'.format(entry_net_inst_key, ni_protocol_id)

                entries.append((entry_protocol_key, _protocol))



                if ni_protocol_id == 'STATIC':

                    static_routes = ni_protocol.get('static-routes', {}).get('static', [])

                    for static_route in static_routes:

                        static_route_prefix = static_route['prefix']



                        next_hops = static_route.get('next-hops', {}).get('next-hop', [])

                        _next_hops = [

                            {

                                'index'  : next_hop['index'],

                                'gateway': next_hop['config']['next-hop'],

                                'metric' : next_hop['config']['metric'],

                            }

                            for next_hop in next_hops

                        ]

                        _next_hops = sorted(_next_hops, key=operator.itemgetter('index'))



                        _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops}

                        entry_static_route_key = '{:s}/static_routes[{:s}]'.format(

                            entry_protocol_key, static_route_prefix

                        )

                        entries.append((entry_static_route_key, _static_route))



            ni_tables = network_instance.get('tables', {}).get('table', [])

            for ni_table in ni_tables:

                ni_table_protocol = ni_table['protocol'].split(':')[-1]

                ni_table_address_family = ni_table['address-family'].split(':')[-1]

                _table = {'protocol': ni_table_protocol, 'address_family': ni_table_address_family}

                entry_table_key = '{:s}/table[{:s},{:s}]'.format(

                    entry_net_inst_key, ni_table_protocol, ni_table_address_family

                )

                entries.append((entry_table_key, _table))



            ni_vlans = network_instance.get('vlans', {}).get('vlan', [])

            for ni_vlan in ni_vlans:

                ni_vlan_id = ni_vlan['vlan-id']



                #ni_vlan_config = ni_vlan['config']

                ni_vlan_state = ni_vlan['state']

                ni_vlan_name = ni_vlan_state['name']



    def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]:

        LOGGER.info('json_data = {:s}'.format(json.dumps(json_data)))

        oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces')

        #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf')))

        response = []

        return response

                _members = [

                    member['state']['interface']

                    for member in ni_vlan.get('members', {}).get('member', [])

                ]

                _vlan = {'vlan_id': ni_vlan_id, 'name': ni_vlan_name, 'members': _members}

                entry_vlan_key = '{:s}/vlan[{:d}]'.format(entry_net_inst_key, ni_vlan_id)

                entries.append((entry_vlan_key, _vlan))



openconfig-network-instance:network-instance
 
 No newline at end of file
 
        return entries