Commit f5199c4c authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Device component: 

- OpenConfigDriver migrate from netconf_client to ncclient library
- Clean integration of Candidate datastore
- OpenConfigDriver code cleanup
parent dcb44d80

src/device/requirements.in

+1 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ fastcache==1.1.0 
grpcio==1.43.0

grpcio-health-checking==1.43.0

Jinja2==3.0.3

netconf-client==2.0.0 #1.7.3

ncclient==0.6.13

p4runtime==1.3.0

paramiko==2.9.2

prometheus-client==0.13.0

src/device/service/drivers/openconfig/OpenConfigDriver.py

+87 −82
Original line number Diff line number Diff line
@@ -20,8 +20,7 @@ from apscheduler.executors.pool import ThreadPoolExecutor 
from apscheduler.job import Job

from apscheduler.jobstores.memory import MemoryJobStore

from apscheduler.schedulers.background import BackgroundScheduler

from netconf_client.connect import connect_ssh, Session

from netconf_client.ncclient import Manager

from ncclient.manager import Manager, connect_ssh

from common.tools.client.RetryDecorator import delay_exponential

from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float

from device.service.driver_api.Exceptions import UnsupportedResourceKeyException
@@ -32,11 +31,9 @@ from .templates import ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filt 
from .RetryDecorator import retry





from ncclient import manager





DEBUG_MODE = True

#logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('ncclient.manager').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('apscheduler.executors.default').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

logging.getLogger('apscheduler.scheduler').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

logging.getLogger('monitoring-client').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)
@@ -65,27 +62,39 @@ class NetconfSessionHandler: 
        self.__port = int(port)

        self.__username       = settings.get('username')

        self.__password       = settings.get('password')

        self.__timeout = int(settings.get('timeout', 120))

        self.__netconf_session : Session = None

        self.__netconf_manager : Manager = None

        self.__key_filename   = settings.get('key_filename')

        self.__hostkey_verify = settings.get('hostkey_verify')

        self.__look_for_keys  = settings.get('look_for_keys')

        self.__allow_agent    = settings.get('allow_agent')

        self.__force_running  = settings.get('force_running', False)

        self.__device_params  = settings.get('device_params')

        self.__manager_params = settings.get('manager_params')

        self.__nc_params      = settings.get('nc_params')

        self.__manager : Manager   = None

        self.__candidate_supported = False



    def connect(self):

        with self.__lock:

            self.__netconf_session = connect_ssh(

                host=self.__address, port=self.__port, username=self.__username, password=self.__password)

            self.__netconf_manager = Manager(self.__netconf_session, timeout=self.__timeout)

            self.__netconf_manager.set_logger_level(logging.DEBUG if DEBUG_MODE else logging.WARNING)

            self.__manager = connect_ssh(

                host=self.__address, port=self.__port, username=self.__username, password=self.__password,

                device_params=self.__device_params, manager_params=self.__manager_params, nc_params=self.__nc_params,

                key_filename=self.__key_filename, hostkey_verify=self.__hostkey_verify, allow_agent=self.__allow_agent,

                look_for_keys=self.__look_for_keys)

            self.__candidate_supported = ':candidate' in self.__manager.server_capabilities

            self.__connected.set()



    def disconnect(self):

        if not self.__connected.is_set(): return

        with self.__lock:

            self.__netconf_manager.close_session()

            self.__manager.close_session()



    @property

    def use_candidate(self): return self.__candidate_supported and not self.__force_running



    @RETRY_DECORATOR

    def get(self, filter=None, with_defaults=None): # pylint: disable=redefined-builtin

        with self.__lock:

            return self.__netconf_manager.get(filter=filter, with_defaults=with_defaults)

            return self.__manager.get(filter=filter, with_defaults=with_defaults)



    @RETRY_DECORATOR

    def edit_config(
@@ -94,10 +103,16 @@ class NetconfSessionHandler: 
    ):

        if config == EMPTY_CONFIG: return

        with self.__lock:

            self.__netconf_manager.edit_config(

            self.__manager.edit_config(

                config, target=target, default_operation=default_operation, test_option=test_option,

                error_option=error_option, format=format)



    def locked(self, target):

        return self.__manager.locked(target=target)



    def commit(self, confirmed=False, timeout=None, persist=None, persist_id=None):

        return self.__manager.commit(confirmed=confirmed, timeout=timeout, persist=persist, persist_id=persist_id)



def compute_delta_sample(previous_sample, previous_timestamp, current_sample, current_timestamp):

    if previous_sample is None: return None

    if previous_timestamp is None: return None
@@ -166,6 +181,36 @@ def do_sampling(samples_cache : SamplesCache, resource_key : str, out_samples : 
    except: # pylint: disable=bare-except

        LOGGER.exception('Error retrieving samples')



def edit_config(

    netconf_handler : NetconfSessionHandler, resources : List[Tuple[str, Any]], delete=False, target='running',

    default_operation='merge', test_option=None, error_option=None, format='xml' # pylint: disable=redefined-builtin

):

    str_method = 'DeleteConfig' if delete else 'SetConfig'

    LOGGER.info('[{:s}] resources = {:s}'.format(str_method, str(resources)))

    results = [None for _ in resources]

    for i,resource in enumerate(resources):

        str_resource_name = 'resources[#{:d}]'.format(i)

        try:

            LOGGER.info('[{:s}] resource = {:s}'.format(str_method, str(resource)))

            chk_type(str_resource_name, resource, (list, tuple))

            chk_length(str_resource_name, resource, min_length=2, max_length=2)

            resource_key,resource_value = resource

            chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

            str_config_message = compose_config(resource_key, resource_value, delete=delete)

            if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

            LOGGER.info('[{:s}] str_config_message[{:d}] = {:s}'.format(

                str_method, len(str_config_message), str(str_config_message)))

            netconf_handler.edit_config(

                config=str_config_message, target=target, default_operation=default_operation,

                test_option=test_option, error_option=error_option, format=format)

            results[i] = True

        except Exception as e: # pylint: disable=broad-except

            str_operation = 'preparing' if target == 'candidate' else ('deleting' if delete else 'setting')

            msg = '[{:s}] Exception {:s} {:s}: {:s}'

            LOGGER.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))

            results[i] = e # if validation fails, store the exception

    return results



class OpenConfigDriver(_Driver):

    def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called

        self.__lock = threading.Lock()
@@ -231,74 +276,34 @@ class OpenConfigDriver(_Driver): 
    def SetConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]:

        chk_type('resources', resources, list)

        if len(resources) == 0: return []

        results = []

        LOGGER.info('[SetConfig] resources = {:s}'.format(str(resources)))

        with self.__lock:

            for i,resource in enumerate(resources):

                str_resource_name = 'resources[#{:d}]'.format(i)

            if self.__netconf_handler.use_candidate:

                with self.__netconf_handler.locked(target='candidate'):

                    results = edit_config(self.__netconf_handler, resources, target='candidate')

                    try:

                    LOGGER.info('[SetConfig] resource = {:s}'.format(str(resource)))

                    chk_type(str_resource_name, resource, (list, tuple))

                    chk_length(str_resource_name, resource, min_length=2, max_length=2)

                    resource_key,resource_value = resource

                    chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

                    str_config_message = compose_config(resource_key, resource_value)

                    

                    with manager.connect(host=address, port=830, username='admin', password='admin',

                                    hostkey_verify=False, device_params={'name':'huaweiyang'},

                                    look_for_keys=False, allow_agent=False) as m:

                        assert(":candidate" in m.server_capabilities)

                        with m.locked(target='candidate'):

                            m.edit_config(config=str_config_message, default_operation="merge", target="candidate")

                            m.commit()

                            results.append(True)

                    """

                    if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

                    LOGGER.info('[SetConfig] str_config_message[{:d}] = {:s}'.format(

                        len(str_config_message), str(str_config_message)))

                    self.__netconf_handler.edit_config(str_config_message, target='running')

                    results.append(True)

                    """

                        self.__netconf_handler.commit()

                    except Exception as e: # pylint: disable=broad-except

                    LOGGER.exception('Exception setting {:s}: {:s}'.format(str_resource_name, str(resource)))

                    results.append(e) # if validation fails, store the exception         

                        LOGGER.exception('[SetConfig] Exception commiting resources: {:s}'.format(str(resources)))

                        results = [e for _ in resources] # if commit fails, set exception in each resource

            else:

                results = edit_config(self.__netconf_handler, resources)

        return results



    def DeleteConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]:

        chk_type('resources', resources, list)

        if len(resources) == 0: return []

        results = []

        LOGGER.info('[DeleteConfig] resources = {:s}'.format(str(resources)))

        with self.__lock:

            for i,resource in enumerate(resources):

                str_resource_name = 'resources[#{:d}]'.format(i)

            if self.__netconf_handler.use_candidate:

                with self.__netconf_handler.locked(target='candidate'):

                    results = edit_config(

                        self.__netconf_handler, resources, target='candidate', delete=True, default_operation='delete')

                    try:

                    LOGGER.info('[DeleteConfig] resource = {:s}'.format(str(resource)))

                    chk_type(str_resource_name, resource, (list, tuple))

                    chk_length(str_resource_name, resource, min_length=2, max_length=2)

                    resource_key,resource_value = resource

                    chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

                    str_config_message = compose_config(resource_key, resource_value, delete=True)

                    

                    with manager.connect(host=address, port=830, username='admin', password='admin',

                                    hostkey_verify=False, device_params={'name':'huaweiyang'},

                                    look_for_keys=False, allow_agent=False) as m:

                        assert(":candidate" in m.server_capabilities)

                        with m.locked(target='candidate'):

                            m.edit_config(config=str_config_message, default_operation="merge", target="candidate")

                            m.commit()

                            results.append(True)

                    

                    """

                    if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

                    LOGGER.info('[DeleteConfig] str_config_message[{:d}] = {:s}'.format(

                        len(str_config_message), str(str_config_message)))

                    self.__netconf_handler.edit_config(str_config_message, target='running')

                    results.append(True)

                    """

                        self.__netconf_handler.commit()

                    except Exception as e: # pylint: disable=broad-except

                    LOGGER.exception('Exception deleting {:s}: {:s}'.format(str_resource_name, str(resource_key)))

                    results.append(e) # if validation fails, store the exception

                        LOGGER.exception('[DeleteConfig] Exception commiting resources: {:s}'.format(str(resources)))

                        results = [e for _ in resources] # if commit fails, set exception in each resource

            else:

                results = edit_config(self.__netconf_handler, resources, delete=True, default_operation='delete')

        return results



    def SubscribeState(self, subscriptions : List[Tuple[str, float, float]]) -> List[Union[bool, Exception]]:

src/device/service/drivers/openconfig/templates/Acl.py

+15 −15
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 
import logging, lxml.etree as ET

from typing import Any, Dict, List, Tuple

from .Namespace import NAMESPACES

from .Tools import add_value_from_collection, add_value_from_tag

from .Tools import add_value_from_tag



LOGGER = logging.getLogger(__name__)

src/device/service/drivers/openconfig/templates/Namespace.py

+2 −3
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 


NAMESPACE_NETCONF                = 'urn:ietf:params:xml:ns:netconf:base:1.0'



NAMESPACE_ACL                    = 'http://openconfig.net/yang/acl'

NAMESPACE_BGP_POLICY             = 'http://openconfig.net/yang/bgp-policy'

NAMESPACE_INTERFACES             = 'http://openconfig.net/yang/interfaces'

NAMESPACE_INTERFACES_IP          = 'http://openconfig.net/yang/interfaces/ip'
@@ -28,8 +29,6 @@ NAMESPACE_POLICY_TYPES_2 = 'http://openconfig.net/yang/policy_types' 
NAMESPACE_ROUTING_POLICY         = 'http://openconfig.net/yang/routing-policy'

NAMESPACE_VLAN                   = 'http://openconfig.net/yang/vlan'



NAMESPACE_ACL                    = 'http://openconfig.net/yang/acl'



NAMESPACES = {

    'nc'   : NAMESPACE_NETCONF,

    'ocacl': NAMESPACE_ACL,

src/device/service/drivers/openconfig/templates/RoutingPolicy.py

+1 −1
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 
import copy, logging, lxml.etree as ET

from typing import Any, Dict, List, Tuple

from .Namespace import NAMESPACES

from .Tools import add_value_from_collection, add_value_from_tag

from .Tools import add_value_from_tag



LOGGER = logging.getLogger(__name__)