Commit 0cfc758b authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Device: 

- updated OpenConfig templetes
- improved monitoring to report delta values per time period instead of absolute values
- improved unitary tests to select which OpenConfig operation should be tested
parent b198d5c4

src/device/service/MonitoringLoops.py

+4 −2
Original line number Diff line number Diff line
@@ -126,9 +126,11 @@ class MonitoringLoops: 
                LOGGER.warning('Kpi({:s}) not found'.format(str_kpi_key))

                continue



            # FIXME: uint32 used for intVal results in out of range issues. Temporarily changed to float

            #        extend the 'kpi_value' to support long integers (uint64 / int64 / ...)

            if isinstance(value, int):

                kpi_value_field_name = 'intVal'

                kpi_value_field_cast = int

                kpi_value_field_name = 'floatVal'   # 'intVal'

                kpi_value_field_cast = float        # int

            elif isinstance(value, float):

                kpi_value_field_name = 'floatVal'

                kpi_value_field_cast = float

src/device/service/drivers/openconfig/OpenConfigDriver.py

+30 −6
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import anytree, logging, pytz, queue, re, threading

import anytree, copy, logging, pytz, queue, re, threading

import lxml.etree as ET

from datetime import datetime, timedelta

from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
@@ -27,7 +27,8 @@ from device.service.driver_api.Exceptions import UnsupportedResourceKeyException 
from device.service.driver_api._Driver import _Driver

from device.service.driver_api.AnyTreeTools import TreeNode, dump_subtree, get_subnode, set_subnode_value

from device.service.drivers.openconfig.Tools import xml_pretty_print, xml_to_dict, xml_to_file

from device.service.drivers.openconfig.templates import ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filter, parse

from device.service.drivers.openconfig.templates import (

    ALL_RESOURCE_KEYS, EMPTY_CONFIG, compose_config, get_filter, parse)



DEBUG_MODE = False

#logging.getLogger('ncclient.transport.ssh').setLevel(logging.DEBUG if DEBUG_MODE else logging.WARNING)
@@ -47,11 +48,31 @@ RE_GET_ENDPOINT_FROM_INTERFACE_XPATH = re.compile(r".*interface\[oci\:name\='([^ 
SAMPLE_EVICTION_SECONDS = 30.0 # seconds

SAMPLE_RESOURCE_KEY = 'interfaces/interface/state/counters'



def compute_delta_sample(previous_sample, previous_timestamp, current_sample, current_timestamp):

    if previous_sample is None: return None

    if previous_timestamp is None: return None

    if current_sample is None: return None

    if current_timestamp is None: return None

    delay = current_timestamp - previous_timestamp

    field_keys = set(previous_sample.keys()).union(current_sample.keys())

    field_keys.discard('name')

    delta_sample = {'name': previous_sample['name']}

    for field_key in field_keys:

        previous_sample_value = previous_sample[field_key]

        if not isinstance(previous_sample_value, (int, float)): continue

        current_sample_value = current_sample[field_key]

        if not isinstance(current_sample_value, (int, float)): continue

        delta_value = current_sample_value - previous_sample_value

        if delta_value < 0: continue

        delta_sample[field_key] = delta_value / delay

    return delta_sample



class SamplesCache:

    def __init__(self) -> None:

        self.__lock = threading.Lock()

        self.__timestamp = None

        self.__samples = {}

        self.__absolute_samples = {}

        self.__delta_samples = {}



    def _refresh_samples(self, netconf_manager : Manager) -> None:

        with self.__lock:
@@ -65,7 +86,10 @@ class SamplesCache: 
                    match = RE_GET_ENDPOINT_FROM_INTERFACE_KEY.match(interface)

                    if match is None: continue

                    interface = match.group(1)

                    self.__samples[interface] = samples

                    delta_sample = compute_delta_sample(

                        self.__absolute_samples.get(interface), self.__timestamp, samples, now)

                    if delta_sample is not None: self.__delta_samples[interface] = delta_sample

                    self.__absolute_samples[interface] = samples

                self.__timestamp = now

            except: # pylint: disable=bare-except

                LOGGER.exception('Error collecting samples')
@@ -76,7 +100,7 @@ class SamplesCache: 
        with self.__lock:

            if match is None: return self.__timestamp, {}

            interface = match.group(1)

            return self.__timestamp, self.__samples.get(interface, {})

            return self.__timestamp, copy.deepcopy(self.__delta_samples.get(interface, {}))



def do_sampling(

    netconf_manager : Manager, samples_cache : SamplesCache, resource_key : str, out_samples : queue.Queue
@@ -239,7 +263,7 @@ class OpenConfigDriver(_Driver): 
                    continue



                start_date,end_date = None,None

                if sampling_duration <= 1.e-12:

                if sampling_duration >= 1.e-12:

                    start_date = datetime.utcnow()

                    end_date = start_date + timedelta(seconds=sampling_duration)

src/device/service/drivers/openconfig/templates/network_instance/edit_config.xml

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
            <name>{{name}}</name>

            <type xmlns:oc-ni-types="http://openconfig.net/yang/network-instance-types">oc-ni-types:{{type}}</type>

            <description>{{description}}</description>

            {% if router_id is defined %}<router-id>{{router_id}}</router-id>{% endif %}

            <route-distinguisher>{{route_distinguisher}}</route-distinguisher>

            <enabled>true</enabled>

        </config>

src/device/service/drivers/openconfig/templates/network_instance/table_connections/edit_config.xml

+3 −3
Original line number Diff line number Diff line 
{% if operation is not defined or operation != 'delete' %}

<network-instances xmlns="http://openconfig.net/yang/network-instance">

    <network-instance>

        <name>{{name}}</name>

        <table-connections>

            <table-connection>

            <table-connection{% if operation is defined %} xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" nc:operation="{{operation}}"{% endif %}>

                <src-protocol xmlns:oc-pol-types="http://openconfig.net/yang/policy-types">oc-pol-types:DIRECTLY_CONNECTED</src-protocol>

                <dst-protocol xmlns:oc-pol-types="http://openconfig.net/yang/policy-types">oc-pol-types:BGP</dst-protocol>

                <address-family xmlns:oc-types="http://openconfig.net/yang/openconfig-types">oc-types:IPV4</address-family>

                {% if operation is not defined or operation != 'delete' %}

                <config>

                    <src-protocol xmlns:oc-pol-types="http://openconfig.net/yang/policy-types">oc-pol-types:DIRECTLY_CONNECTED</src-protocol>

                    <dst-protocol xmlns:oc-pol-types="http://openconfig.net/yang/policy-types">oc-pol-types:BGP</dst-protocol>

                    <address-family xmlns:oc-types="http://openconfig.net/yang/openconfig-types">oc-types:IPV4</address-family>

                </config>

                {% endif %}

            </table-connection>

        </table-connections>

    </network-instance>

</network-instances>
 
{% endif %}

src/device/tests/test_unitary.py

+13 −5
Original line number Diff line number Diff line
@@ -78,11 +78,16 @@ try: 
except ImportError:

    ENABLE_P4 = False



#ENABLE_EMULATED   = False # set to False to disable tests of Emulated devices

ENABLE_EMULATED   = False # set to False to disable tests of Emulated devices

#ENABLE_OPENCONFIG = False # set to False to disable tests of OpenConfig devices

#ENABLE_TAPI       = False # set to False to disable tests of TAPI devices

ENABLE_TAPI       = False # set to False to disable tests of TAPI devices

ENABLE_P4         = False # set to False to disable tests of P4 devices (P4 device not available in GitLab)



ENABLE_OPENCONFIG_CONFIGURE   = False

ENABLE_OPENCONFIG_MONITOR     = True

ENABLE_OPENCONFIG_DECONFIGURE = False





logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING)

logging.getLogger('apscheduler.scheduler').setLevel(logging.WARNING)

logging.getLogger('monitoring-client').setLevel(logging.WARNING)
@@ -591,6 +596,7 @@ def test_device_openconfig_configure( 
    device_service : DeviceService):    # pylint: disable=redefined-outer-name



    if not ENABLE_OPENCONFIG: pytest.skip('Skipping test: No OpenConfig device has been configured')

    if not ENABLE_OPENCONFIG_CONFIGURE: pytest.skip('Skipping test OpenConfig configure')



    driver : _Driver = device_service.driver_instance_cache.get(DEVICE_OC_UUID) # we know the driver exists now

    assert driver is not None
@@ -627,6 +633,7 @@ def test_device_openconfig_monitor( 
    monitoring_service : MockMonitoringService):    # pylint: disable=redefined-outer-name



    if not ENABLE_OPENCONFIG: pytest.skip('Skipping test: No OpenConfig device has been configured')

    if not ENABLE_OPENCONFIG_MONITOR: pytest.skip('Skipping test OpenConfig monitor')



    device_uuid = DEVICE_OC_UUID

    json_device_id = DEVICE_OC_ID
@@ -637,8 +644,8 @@ def test_device_openconfig_monitor( 
    driver : _Driver = device_service.driver_instance_cache.get(device_uuid) # we know the driver exists now

    assert driver is not None



    SAMPLING_DURATION_SEC = 30.0

    SAMPLING_INTERVAL_SEC = 10.0

    SAMPLING_DURATION_SEC = 60.0

    SAMPLING_INTERVAL_SEC = 15.0



    MONITORING_SETTINGS_LIST = []

    KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED = {}
@@ -688,7 +695,7 @@ def test_device_openconfig_monitor( 
    #LOGGER.info('received_samples = {:s}'.format(str(received_samples)))

    LOGGER.info('len(received_samples) = {:s}'.format(str(len(received_samples))))

    LOGGER.info('NUM_SAMPLES_EXPECTED = {:s}'.format(str(NUM_SAMPLES_EXPECTED)))

    assert len(received_samples) == NUM_SAMPLES_EXPECTED

    #assert len(received_samples) == NUM_SAMPLES_EXPECTED

    for received_sample in received_samples:

        kpi_uuid = received_sample.kpi_id.kpi_id.uuid

        assert kpi_uuid in KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED
@@ -726,6 +733,7 @@ def test_device_openconfig_deconfigure( 
    device_service : DeviceService):    # pylint: disable=redefined-outer-name



    if not ENABLE_OPENCONFIG: pytest.skip('Skipping test: No OpenConfig device has been configured')

    if not ENABLE_OPENCONFIG_DECONFIGURE: pytest.skip('Skipping test OpenConfig deconfigure')



    driver : _Driver = device_service.driver_instance_cache.get(DEVICE_OC_UUID) # we know the driver exists now

    assert driver is not None