Commit 5ebb9dcb authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/device-oc-acl-candidate' into 'develop' 

Device component: Added support for ACLs and Candidate datastore in OpenConfigDriver

See merge request teraflow-h2020/controller!102
parents f83b23a5 bc601571

src/device/requirements.in

+1 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ fastcache==1.1.0 
grpcio==1.43.0

grpcio-health-checking==1.43.0

Jinja2==3.0.3

netconf-client==2.0.0 #1.7.3

ncclient==0.6.13

p4runtime==1.3.0

paramiko==2.9.2

prometheus-client==0.13.0

src/device/service/driver_api/_Driver.py

+1 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ RESOURCE_ENDPOINTS = '__endpoints__' 
RESOURCE_INTERFACES        = '__interfaces__'

RESOURCE_NETWORK_INSTANCES = '__network_instances__'

RESOURCE_ROUTING_POLICIES  = '__routing_policies__'

RESOURCE_ACL               = '__acl__'



class _Driver:

    def __init__(self, address : str, port : int, **settings) -> None:

src/device/service/drivers/openconfig/OpenConfigDriver.py

+84 −53
Original line number Diff line number Diff line
@@ -20,8 +20,7 @@ from apscheduler.executors.pool import ThreadPoolExecutor 
from apscheduler.job import Job

from apscheduler.jobstores.memory import MemoryJobStore

from apscheduler.schedulers.background import BackgroundScheduler

from netconf_client.connect import connect_ssh, Session

from netconf_client.ncclient import Manager

from ncclient.manager import Manager, connect_ssh

from common.tools.client.RetryDecorator import delay_exponential

from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float

from device.service.driver_api.Exceptions import UnsupportedResourceKeyException
@@ -31,8 +30,10 @@ from device.service.driver_api.AnyTreeTools import TreeNode, get_subnode, set_su 
from .templates import ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filter, parse

from .RetryDecorator import retry





DEBUG_MODE = False

#logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('ncclient.manager').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)

logging.getLogger('apscheduler.executors.default').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

logging.getLogger('apscheduler.scheduler').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)

logging.getLogger('monitoring-client').setLevel(logging.INFO if DEBUG_MODE else logging.ERROR)
@@ -61,27 +62,39 @@ class NetconfSessionHandler: 
        self.__port = int(port)

        self.__username       = settings.get('username')

        self.__password       = settings.get('password')

        self.__timeout = int(settings.get('timeout', 120))

        self.__netconf_session : Session = None

        self.__netconf_manager : Manager = None

        self.__key_filename   = settings.get('key_filename')

        self.__hostkey_verify = settings.get('hostkey_verify', True)

        self.__look_for_keys  = settings.get('look_for_keys', True)

        self.__allow_agent    = settings.get('allow_agent', True)

        self.__force_running  = settings.get('force_running', False)

        self.__device_params  = settings.get('device_params', {})

        self.__manager_params = settings.get('manager_params', {})

        self.__nc_params      = settings.get('nc_params', {})

        self.__manager : Manager   = None

        self.__candidate_supported = False



    def connect(self):

        with self.__lock:

            self.__netconf_session = connect_ssh(

                host=self.__address, port=self.__port, username=self.__username, password=self.__password)

            self.__netconf_manager = Manager(self.__netconf_session, timeout=self.__timeout)

            self.__netconf_manager.set_logger_level(logging.DEBUG if DEBUG_MODE else logging.WARNING)

            self.__manager = connect_ssh(

                host=self.__address, port=self.__port, username=self.__username, password=self.__password,

                device_params=self.__device_params, manager_params=self.__manager_params, nc_params=self.__nc_params,

                key_filename=self.__key_filename, hostkey_verify=self.__hostkey_verify, allow_agent=self.__allow_agent,

                look_for_keys=self.__look_for_keys)

            self.__candidate_supported = ':candidate' in self.__manager.server_capabilities

            self.__connected.set()



    def disconnect(self):

        if not self.__connected.is_set(): return

        with self.__lock:

            self.__netconf_manager.close_session()

            self.__manager.close_session()



    @property

    def use_candidate(self): return self.__candidate_supported and not self.__force_running



    @RETRY_DECORATOR

    def get(self, filter=None, with_defaults=None): # pylint: disable=redefined-builtin

        with self.__lock:

            return self.__netconf_manager.get(filter=filter, with_defaults=with_defaults)

            return self.__manager.get(filter=filter, with_defaults=with_defaults)



    @RETRY_DECORATOR

    def edit_config(
@@ -90,10 +103,16 @@ class NetconfSessionHandler: 
    ):

        if config == EMPTY_CONFIG: return

        with self.__lock:

            self.__netconf_manager.edit_config(

            self.__manager.edit_config(

                config, target=target, default_operation=default_operation, test_option=test_option,

                error_option=error_option, format=format)



    def locked(self, target):

        return self.__manager.locked(target=target)



    def commit(self, confirmed=False, timeout=None, persist=None, persist_id=None):

        return self.__manager.commit(confirmed=confirmed, timeout=timeout, persist=persist, persist_id=persist_id)



def compute_delta_sample(previous_sample, previous_timestamp, current_sample, current_timestamp):

    if previous_sample is None: return None

    if previous_timestamp is None: return None
@@ -162,6 +181,36 @@ def do_sampling(samples_cache : SamplesCache, resource_key : str, out_samples : 
    except: # pylint: disable=bare-except

        LOGGER.exception('Error retrieving samples')



def edit_config(

    netconf_handler : NetconfSessionHandler, resources : List[Tuple[str, Any]], delete=False, target='running',

    default_operation='merge', test_option=None, error_option=None, format='xml' # pylint: disable=redefined-builtin

):

    str_method = 'DeleteConfig' if delete else 'SetConfig'

    LOGGER.info('[{:s}] resources = {:s}'.format(str_method, str(resources)))

    results = [None for _ in resources]

    for i,resource in enumerate(resources):

        str_resource_name = 'resources[#{:d}]'.format(i)

        try:

            LOGGER.info('[{:s}] resource = {:s}'.format(str_method, str(resource)))

            chk_type(str_resource_name, resource, (list, tuple))

            chk_length(str_resource_name, resource, min_length=2, max_length=2)

            resource_key,resource_value = resource

            chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

            str_config_message = compose_config(resource_key, resource_value, delete=delete)

            if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

            LOGGER.info('[{:s}] str_config_message[{:d}] = {:s}'.format(

                str_method, len(str_config_message), str(str_config_message)))

            netconf_handler.edit_config(

                config=str_config_message, target=target, default_operation=default_operation,

                test_option=test_option, error_option=error_option, format=format)

            results[i] = True

        except Exception as e: # pylint: disable=broad-except

            str_operation = 'preparing' if target == 'candidate' else ('deleting' if delete else 'setting')

            msg = '[{:s}] Exception {:s} {:s}: {:s}'

            LOGGER.exception(msg.format(str_method, str_operation, str_resource_name, str(resource)))

            results[i] = e # if validation fails, store the exception

    return results



class OpenConfigDriver(_Driver):

    def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called

        self.__lock = threading.Lock()
@@ -227,51 +276,33 @@ class OpenConfigDriver(_Driver): 
    def SetConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]:

        chk_type('resources', resources, list)

        if len(resources) == 0: return []

        results = []

        LOGGER.info('[SetConfig] resources = {:s}'.format(str(resources)))

        with self.__lock:

            for i,resource in enumerate(resources):

                str_resource_name = 'resources[#{:d}]'.format(i)

            if self.__netconf_handler.use_candidate:

                with self.__netconf_handler.locked(target='candidate'):

                    results = edit_config(self.__netconf_handler, resources, target='candidate')

                    try:

                    LOGGER.info('[SetConfig] resource = {:s}'.format(str(resource)))

                    chk_type(str_resource_name, resource, (list, tuple))

                    chk_length(str_resource_name, resource, min_length=2, max_length=2)

                    resource_key,resource_value = resource

                    chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

                    str_config_message = compose_config(resource_key, resource_value)

                    if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

                    LOGGER.info('[SetConfig] str_config_message[{:d}] = {:s}'.format(

                        len(str_config_message), str(str_config_message)))

                    self.__netconf_handler.edit_config(str_config_message, target='running')

                    results.append(True)

                        self.__netconf_handler.commit()

                    except Exception as e: # pylint: disable=broad-except

                    LOGGER.exception('Exception setting {:s}: {:s}'.format(str_resource_name, str(resource)))

                    results.append(e) # if validation fails, store the exception

                        LOGGER.exception('[SetConfig] Exception commiting resources: {:s}'.format(str(resources)))

                        results = [e for _ in resources] # if commit fails, set exception in each resource

            else:

                results = edit_config(self.__netconf_handler, resources)

        return results



    def DeleteConfig(self, resources : List[Tuple[str, Any]]) -> List[Union[bool, Exception]]:

        chk_type('resources', resources, list)

        if len(resources) == 0: return []

        results = []

        LOGGER.info('[DeleteConfig] resources = {:s}'.format(str(resources)))

        with self.__lock:

            for i,resource in enumerate(resources):

                str_resource_name = 'resources[#{:d}]'.format(i)

            if self.__netconf_handler.use_candidate:

                with self.__netconf_handler.locked(target='candidate'):

                    results = edit_config(self.__netconf_handler, resources, target='candidate', delete=True)

                    try:

                    LOGGER.info('[DeleteConfig] resource = {:s}'.format(str(resource)))

                    chk_type(str_resource_name, resource, (list, tuple))

                    chk_length(str_resource_name, resource, min_length=2, max_length=2)

                    resource_key,resource_value = resource

                    chk_string(str_resource_name + '.key', resource_key, allow_empty=False)

                    str_config_message = compose_config(resource_key, resource_value, delete=True)

                    if str_config_message is None: raise UnsupportedResourceKeyException(resource_key)

                    LOGGER.info('[DeleteConfig] str_config_message[{:d}] = {:s}'.format(

                        len(str_config_message), str(str_config_message)))

                    self.__netconf_handler.edit_config(str_config_message, target='running')

                    results.append(True)

                        self.__netconf_handler.commit()

                    except Exception as e: # pylint: disable=broad-except

                    LOGGER.exception('Exception deleting {:s}: {:s}'.format(str_resource_name, str(resource_key)))

                    results.append(e) # if validation fails, store the exception

                        LOGGER.exception('[DeleteConfig] Exception commiting resources: {:s}'.format(str(resources)))

                        results = [e for _ in resources] # if commit fails, set exception in each resource

            else:

                results = edit_config(self.__netconf_handler, resources, delete=True)

        return results



    def SubscribeState(self, subscriptions : List[Tuple[str, float, float]]) -> List[Union[bool, Exception]]:

src/device/service/drivers/openconfig/templates/Acl.py

0 → 100644
+123 −0
Original line number Diff line number Diff line 
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import logging, lxml.etree as ET

from typing import Any, Dict, List, Tuple

from .Namespace import NAMESPACES

from .Tools import add_value_from_tag



LOGGER = logging.getLogger(__name__)



XPATH_ACL_SET     = "//ocacl:acl/ocacl:acl-sets/ocacl:acl-set"

XPATH_A_ACL_ENTRY = ".//ocacl:acl-entries/ocacl:ecl-entry"

XPATH_A_IPv4      = ".//ocacl:ipv4/ocacl:config"

XPATH_A_TRANSPORT = ".//ocacl:transport/ocacl:config"

XPATH_A_ACTIONS   = ".//ocacl:actions/ocacl:config"



XPATH_INTERFACE   = "//ocacl:acl/ocacl:interfaces/ocacl:interface"

XPATH_I_INGRESS   = ".//ocacl:ingress-acl-sets/ocacl:ingress-acl-set"

XPATH_I_EGRESS    = ".//ocacl:egress-acl-sets/ocacl:egress-acl-set"



def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:

    #LOGGER.info('[ACL] xml_data = {:s}'.format(str(ET.tostring(xml_data))))



    response = []

    acl = {}



    for xml_acl in xml_data.xpath(XPATH_ACL_SET, namespaces=NAMESPACES):

        #LOGGER.info('xml_acl = {:s}'.format(str(ET.tostring(xml_acl))))



        acl_name = xml_acl.find('ocacl:name', namespaces=NAMESPACES)

        if acl_name is None or acl_name.text is None: continue

        add_value_from_tag(acl, 'name', acl_name)



        acl_type = xml_acl.find('ocacl:type', namespaces=NAMESPACES)

        add_value_from_tag(acl, 'type', acl_type)



        for xml_acl_entries in xml_acl.xpath(XPATH_A_ACL_ENTRY, namespaces=NAMESPACES):



            acl_id = xml_acl_entries.find('ocacl:sequence_id', namespaces=NAMESPACES)

            add_value_from_tag(acl, 'sequence_id', acl_id)



            for xml_ipv4 in xml_acl_entries.xpath(XPATH_A_IPv4, namespaces=NAMESPACES):



                ipv4_source = xml_ipv4.find('ocacl:source_address', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'source_address' , ipv4_source)



                ipv4_destination = xml_ipv4.find('ocacl:destination_address', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'destination_address' , ipv4_destination)



                ipv4_protocol = xml_ipv4.find('ocacl:protocol', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'protocol' , ipv4_protocol)



                ipv4_dscp = xml_ipv4.find('ocacl:dscp', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'dscp' , ipv4_dscp)



                ipv4_hop_limit = xml_ipv4.find('ocacl:hop_limit', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'hop_limit' , ipv4_hop_limit)



            for xml_transport in xml_acl_entries.xpath(XPATH_A_TRANSPORT, namespaces=NAMESPACES):



                transport_source = xml_transport.find('ocacl:source_port', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'source_port' ,transport_source)



                transport_destination = xml_transport.find('ocacl:destination_port', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'destination_port' ,transport_destination)



                transport_tcp_flags = xml_transport.find('ocacl:tcp_flags', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'tcp_flags' ,transport_tcp_flags)



            for xml_action in xml_acl_entries.xpath(XPATH_A_ACTIONS, namespaces=NAMESPACES):



                action = xml_action.find('ocacl:forwarding_action', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'forwarding_action' ,action)



                log_action = xml_action.find('ocacl:log_action', namespaces=NAMESPACES)

                add_value_from_tag(acl, 'log_action' ,log_action)



            resource_key =  '/acl/acl-set[{:s}][{:s}]/acl-entry[{:s}]'.format(

                acl['name'], acl['type'], acl['sequence-id'])

            response.append((resource_key,acl))



    for xml_interface in xml_data.xpath(XPATH_INTERFACE, namespaces=NAMESPACES):



        interface = {}



        interface_id = xml_interface.find('ocacl:id', namespaces=NAMESPACES)

        add_value_from_tag(interface, 'id' , interface_id)



        for xml_ingress in xml_interface.xpath(XPATH_I_INGRESS, namespaces=NAMESPACES):



            i_name = xml_ingress.find('ocacl:set_name_ingress', namespaces=NAMESPACES)

            add_value_from_tag(interface, 'ingress_set_name' , i_name)



            i_type = xml_ingress.find('ocacl:type_ingress', namespaces=NAMESPACES)

            add_value_from_tag(interface, 'ingress_type' , i_type)



            resource_key =  '/acl/interfaces/ingress[{:s}][{:s}]'.format(

                acl['name'], acl['type'])

            response.append((resource_key,interface))



        for xml_egress in xml_interface.xpath(XPATH_I_EGRESS, namespaces=NAMESPACES):



            e_name = xml_egress.find('ocacl:set_name_egress', namespaces=NAMESPACES)

            add_value_from_tag(interface, 'egress_set_name' , e_name)



            e_type = xml_egress.find('ocacl:type_egress', namespaces=NAMESPACES)

            add_value_from_tag(interface, 'egress_type' , e_type)



            resource_key =  '/acl/interfaces/egress[{:s}][{:s}]'.format(

                acl['name'], acl['type'])

            response.append((resource_key,interface))

    return response

src/device/service/drivers/openconfig/templates/Namespace.py

+3 −1
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 


NAMESPACE_NETCONF                = 'urn:ietf:params:xml:ns:netconf:base:1.0'



NAMESPACE_ACL                    = 'http://openconfig.net/yang/acl'

NAMESPACE_BGP_POLICY             = 'http://openconfig.net/yang/bgp-policy'

NAMESPACE_INTERFACES             = 'http://openconfig.net/yang/interfaces'

NAMESPACE_INTERFACES_IP          = 'http://openconfig.net/yang/interfaces/ip'
@@ -30,6 +31,7 @@ NAMESPACE_VLAN = 'http://openconfig.net/yang/vlan' 


NAMESPACES = {

    'nc'   : NAMESPACE_NETCONF,

    'ocacl': NAMESPACE_ACL,

    'ocbp' : NAMESPACE_BGP_POLICY,

    'oci'  : NAMESPACE_INTERFACES,

    'ociip': NAMESPACE_INTERFACES_IP,