Commit f619d55d authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

First working version of Device Monitoring including Monitoring of Emulated devices.

parent d937aecd

src/device/service/drivers/openconfig/templates/EndPoints.py

+12 −2
Original line number Diff line number Diff line 
import logging, lxml.etree as ET

from typing import Any, Dict, List, Tuple

from device.service.database.KpiSampleType import ORM_KpiSampleType

from .Namespace import NAMESPACES

from .Tools import add_value_from_tag

from .Tools import add_value_from_collection, add_value_from_tag



LOGGER = logging.getLogger(__name__)



XPATH_PORTS = "//ocp:components/ocp:component/ocp:state[ocp:type='PORT']/.."

XPATH_INTERFACE_COUNTER = "//oci:interfaces/oci:interface[oci:name='{:s}']/state/counters/{:s}"



def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:

    response = []
@@ -22,6 +24,14 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: 
            'ocpp:port/ocpp:breakout-mode/ocpp:state/ocpp:channel-speed', namespaces=NAMESPACES)

        add_value_from_tag(endpoint, 'type', component_type)



        sample_types = {

            ORM_KpiSampleType.BYTES_RECEIVED.value     : XPATH_INTERFACE_COUNTER.format(endpoint['uuid'], 'in-octets' ),

            ORM_KpiSampleType.BYTES_TRANSMITTED.value  : XPATH_INTERFACE_COUNTER.format(endpoint['uuid'], 'out-octets'),

            ORM_KpiSampleType.PACKETS_RECEIVED.value   : XPATH_INTERFACE_COUNTER.format(endpoint['uuid'], 'in-pkts'   ),

            ORM_KpiSampleType.PACKETS_TRANSMITTED.value: XPATH_INTERFACE_COUNTER.format(endpoint['uuid'], 'out-pkts'  ),

        }

        add_value_from_collection(endpoint, 'sample_types', sample_types)



        if len(endpoint) == 0: continue

        response.append(('endpoint[{:s}]'.format(endpoint['name']), endpoint))

        response.append(('endpoint[{:s}]'.format(endpoint['uuid']), endpoint))

    return response

src/device/tests/Device_Emulated.py

+50 −60
Original line number Diff line number Diff line 
import operator

from copy import deepcopy

from device.proto.context_pb2 import DeviceDriverEnum, DeviceOperationalStatusEnum

from device.service.database.KpiSampleType import ORM_KpiSampleType
@@ -22,32 +23,29 @@ DEVICE_EMU = { 
}



PACKET_PORT_SAMPLE_TYPES = [

    ORM_KpiSampleType.PACKETS_TRANSMITTED.value,

    ORM_KpiSampleType.PACKETS_RECEIVED.value,

    ORM_KpiSampleType.BYTES_TRANSMITTED.value,

    ORM_KpiSampleType.BYTES_RECEIVED.value,

    ORM_KpiSampleType.PACKETS_TRANSMITTED,

    ORM_KpiSampleType.PACKETS_RECEIVED,

    ORM_KpiSampleType.BYTES_TRANSMITTED,

    ORM_KpiSampleType.BYTES_RECEIVED,

]



DEVICE_EMU_ENDPOINTS = [

    ('EP1', '10Gbps', PACKET_PORT_SAMPLE_TYPES),

    ('EP2', '10Gbps', PACKET_PORT_SAMPLE_TYPES),

    ('EP3', '10Gbps', PACKET_PORT_SAMPLE_TYPES),

    ('EP4', '10Gbps', PACKET_PORT_SAMPLE_TYPES),

]

ENDPOINT_UUIDS = ['EP1', 'EP2', 'EP3', 'EP4']



DEVICE_EMU_ENDPOINTS = []

for endpoint_uuid in ENDPOINT_UUIDS:

    DEVICE_EMU_ENDPOINTS.append((endpoint_uuid, '10Gbps', PACKET_PORT_SAMPLE_TYPES))



RSRC_EP       = '/endpoints/endpoint[{}]'

RSRC_SUBIF    = RSRC_EP    + '/subinterfaces/subinterface[{}]'

RSRC_ADDRIPV4 = RSRC_SUBIF + '/ipv4/address[{}]'

RSRC_EP       = '/endpoints/endpoint[{:s}]'

RSRC_SUBIF    = RSRC_EP    + '/subinterfaces/subinterface[{:d}]'

RSRC_ADDRIPV4 = RSRC_SUBIF + '/ipv4/address[{:s}]'



DEVICE_EMU_ENDPOINTS_COOKED = []

for endpoint_uuid,endpoint_type,endpoint_sample_types in DEVICE_EMU_ENDPOINTS:

    endpoint_resource_key = RSRC_EP.format(str(endpoint_uuid))

    sample_types = {

        ORM_KpiSampleType.PACKETS_TRANSMITTED.value: '{:s}/state/packets_transmitted'.format(endpoint_resource_key),

        ORM_KpiSampleType.PACKETS_RECEIVED   .value: '{:s}/state/packets_received'   .format(endpoint_resource_key),

        ORM_KpiSampleType.BYTES_TRANSMITTED  .value: '{:s}/state/bytes_transmitted'  .format(endpoint_resource_key),

        ORM_KpiSampleType.BYTES_RECEIVED     .value: '{:s}/state/bytes_received'     .format(endpoint_resource_key),

    }

    sample_types = {}

    for endpoint_sample_type in endpoint_sample_types:

        sample_type_name = endpoint_sample_type.name.lower()

        sample_types[endpoint_sample_type.value] = '{:s}/state/{:s}'.format(endpoint_resource_key, sample_type_name)

    endpoint_resource_value = {'uuid': endpoint_uuid, 'type': endpoint_type, 'sample_types': sample_types}

    DEVICE_EMU_ENDPOINTS_COOKED.append((endpoint_resource_key, endpoint_resource_value))
@@ -55,31 +53,30 @@ DEVICE_EMU_CONNECT_RULES = [ 
    config_rule_set('_connect/address',  DEVICE_EMU_ADDRESS ),

    config_rule_set('_connect/port',     DEVICE_EMU_PORT    ),

    config_rule_set('_connect/settings', {'endpoints': [

        {'uuid': endpoint_uuid, 'type': endpoint_type, 'sample_types': endpoint_sample_types}

        {

            'uuid': endpoint_uuid, 'type': endpoint_type,

            'sample_types': list(map(operator.attrgetter('value'), endpoint_sample_types)),

        }

        for endpoint_uuid,endpoint_type,endpoint_sample_types in DEVICE_EMU_ENDPOINTS

    ]}),

]



DEVICE_EMU_CONFIG_ENDPOINTS = [

    config_rule_set(RSRC_EP.format('EP1'), {'enabled' : True}),

    config_rule_set(RSRC_EP.format('EP2'), {'enabled' : True}),

    config_rule_set(RSRC_EP.format('EP3'), {'enabled' : True}),

    config_rule_set(RSRC_EP.format('EP4'), {'enabled' : True}),

]



DEVICE_EMU_CONFIG_ADDRESSES = [

    config_rule_set(RSRC_SUBIF   .format('EP1', 0            ), {'index': 0}),

    config_rule_set(RSRC_ADDRIPV4.format('EP1', 0, '10.1.0.1'), {'ip': '10.1.0.1', 'prefix_length': 24}),



    config_rule_set(RSRC_SUBIF   .format('EP2', 0            ), {'index': 0}),

    config_rule_set(RSRC_ADDRIPV4.format('EP2', 0, '10.2.0.1'), {'ip': '10.2.0.1', 'prefix_length': 24}),



    config_rule_set(RSRC_SUBIF   .format('EP3', 0            ), {'index': 0}),

    config_rule_set(RSRC_ADDRIPV4.format('EP3', 0, '10.3.0.1'), {'ip': '10.3.0.1', 'prefix_length': 24}),



    config_rule_set(RSRC_SUBIF   .format('EP4', 0            ), {'index': 0}),

    config_rule_set(RSRC_ADDRIPV4.format('EP4', 0, '10.4.0.1'), {'ip': '10.4.0.1', 'prefix_length': 24}),

]

DEVICE_EMU_CONFIG_ENDPOINTS = []

for endpoint_uuid in ENDPOINT_UUIDS:

    DEVICE_EMU_CONFIG_ENDPOINTS.append(config_rule_set(RSRC_EP.format(endpoint_uuid), {'enabled' : True}))



DEVICE_EMU_CONFIG_ADDRESSES = []

for endpoint_uuid in ENDPOINT_UUIDS:

    endpoint_number = int(endpoint_uuid.replace('EP', ''))

    subinterface_index = 0

    subinterface_address = '10.{:d}.{:d}.1'.format(endpoint_number, subinterface_index)

    subinterface_prefix_length = 24

    DEVICE_EMU_CONFIG_ADDRESSES.extend([

        config_rule_set(RSRC_SUBIF   .format(endpoint_uuid, subinterface_index), {

            'index': subinterface_index}),

        config_rule_set(RSRC_ADDRIPV4.format(endpoint_uuid, subinterface_index, subinterface_address), {

            'ip': subinterface_address, 'prefix_length': subinterface_prefix_length}),

    ])



DEVICE_EMU_RECONFIG_ADDRESSES = [

    config_rule_delete(RSRC_SUBIF   .format('EP2', 0            ), {}),
@@ -89,23 +86,16 @@ DEVICE_EMU_RECONFIG_ADDRESSES = [ 
    config_rule_set   (RSRC_ADDRIPV4.format('EP2', 1, '10.2.1.1'), {'ip': '10.2.1.1', 'prefix_length': 24}),

]



DEVICE_EMU_DECONFIG_ADDRESSES = [

    config_rule_delete(RSRC_SUBIF   .format('EP1', 0            ), {}),

    config_rule_delete(RSRC_ADDRIPV4.format('EP1', 0, '10.1.0.1'), {}),



    config_rule_delete(RSRC_SUBIF   .format('EP2', 1            ), {}),

    config_rule_delete(RSRC_ADDRIPV4.format('EP2', 1, '10.2.1.1'), {}),



    config_rule_delete(RSRC_SUBIF   .format('EP3', 0            ), {}),

    config_rule_delete(RSRC_ADDRIPV4.format('EP3', 0, '10.3.0.1'), {}),



    config_rule_delete(RSRC_SUBIF   .format('EP4', 0            ), {}),

    config_rule_delete(RSRC_ADDRIPV4.format('EP4', 0, '10.4.0.1'), {}),

]



DEVICE_EMU_DECONFIG_ENDPOINTS = [

    config_rule_delete(RSRC_EP.format('EP1'), {}),

    config_rule_delete(RSRC_EP.format('EP2'), {}),

    config_rule_delete(RSRC_EP.format('EP3'), {}),

    config_rule_delete(RSRC_EP.format('EP4'), {}),

]
 
DEVICE_EMU_DECONFIG_ADDRESSES = []

for endpoint_uuid in ENDPOINT_UUIDS:

    endpoint_number = int(endpoint_uuid.replace('EP', ''))

    subinterface_index = 1 if endpoint_uuid == 'EP2' else 0

    subinterface_address = '10.{:d}.{:d}.1'.format(endpoint_number, subinterface_index)

    DEVICE_EMU_DECONFIG_ADDRESSES.extend([

        config_rule_delete(RSRC_SUBIF   .format(endpoint_uuid, subinterface_index), {}),

        config_rule_delete(RSRC_ADDRIPV4.format(endpoint_uuid, subinterface_index, subinterface_address), {}),

    ])



DEVICE_EMU_DECONFIG_ENDPOINTS = []

for endpoint_uuid in ENDPOINT_UUIDS:

    DEVICE_EMU_DECONFIG_ENDPOINTS.append(config_rule_delete(RSRC_EP.format(endpoint_uuid), {}))

src/device/tests/Device_OpenConfig_Template.py

+15 −11
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ DEVICE_OC_ADDRESS = '127.0.0.1' # populate the Netconf Server IP address 
DEVICE_OC_PORT     = '830'        # populate the Netconf Server port of the device to test

DEVICE_OC_USERNAME = 'username'   # populate the Netconf Server username of the device to test

DEVICE_OC_PASSWORD = 'password'   # populate the Netconf Server password of the device to test

DEVICE_OC_TIMEOUT  = 120

DEVICE_OC_DRIVERS  = [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG]



DEVICE_OC_ID = {'device_uuid': {'uuid': DEVICE_OC_UUID}}
@@ -25,8 +26,11 @@ DEVICE_OC = { 
DEVICE_OC_CONNECT_RULES = [

    config_rule_set('_connect/address', DEVICE_OC_ADDRESS),

    config_rule_set('_connect/port',    DEVICE_OC_PORT   ),

    config_rule_set('_connect/username', DEVICE_OC_USERNAME),

    config_rule_set('_connect/password', DEVICE_OC_PASSWORD),

    config_rule_set('_connect/settings', {

        'username': DEVICE_OC_USERNAME,

        'password': DEVICE_OC_PASSWORD,

        'timeout' : DEVICE_OC_TIMEOUT,

    }),

]



DEVICE_OC_CONFIG_RULES   = []           # populate your configuration rules to test

src/device/tests/test_unitary.py

+43 −28
Original line number Diff line number Diff line
@@ -34,8 +34,8 @@ from monitoring.client.monitoring_client import MonitoringClient 
from .CommonObjects import CONTEXT, TOPOLOGY

from .Device_Emulated import (

    DEVICE_EMU, DEVICE_EMU_CONFIG_ADDRESSES, DEVICE_EMU_CONFIG_ENDPOINTS, DEVICE_EMU_CONNECT_RULES,

    DEVICE_EMU_DECONFIG_ADDRESSES, DEVICE_EMU_DECONFIG_ENDPOINTS, DEVICE_EMU_ENDPOINTS_COOKED, DEVICE_EMU_ID,

    DEVICE_EMU_RECONFIG_ADDRESSES, DEVICE_EMU_UUID)

    DEVICE_EMU_DECONFIG_ADDRESSES, DEVICE_EMU_DECONFIG_ENDPOINTS, DEVICE_EMU_ENDPOINTS, DEVICE_EMU_ENDPOINTS_COOKED,

    DEVICE_EMU_ID, DEVICE_EMU_RECONFIG_ADDRESSES, DEVICE_EMU_UUID)

try:

    from .Device_OpenConfig_Infinera import(

        DEVICE_OC, DEVICE_OC_CONFIG_RULES, DEVICE_OC_DECONFIG_RULES, DEVICE_OC_CONNECT_RULES, DEVICE_OC_ID,
@@ -316,28 +316,36 @@ def test_device_emulated_monitor( 


    SAMPLING_DURATION_SEC = 3.0

    SAMPLING_INTERVAL_SEC = 0.5

    NUM_SAMPLES_EXPECTED = SAMPLING_DURATION_SEC / SAMPLING_INTERVAL_SEC



    ENDPOINT_UUID = 'EP2'

    SAMPLE_TYPE = KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED

    SAMPLE_TYPE_NAME = str(KpiSampleType.Name(SAMPLE_TYPE)).upper().replace('KPISAMPLETYPE_', '')

    KPI_UUID = '{:s}-{:s}-{:s}-kpi_uuid'.format(DEVICE_EMU_UUID, ENDPOINT_UUID, str(SAMPLE_TYPE))

    MONITORING_SETTINGS = {

        'kpi_id'        : {'kpi_id': {'uuid': KPI_UUID}},



    MONITORING_SETTINGS_LIST = []

    KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED = {}

    for endpoint_uuid,_,sample_types in DEVICE_EMU_ENDPOINTS:

        for sample_type in sample_types:

            sample_type_id = sample_type.value

            sample_type_name = str(KpiSampleType.Name(sample_type_id)).upper().replace('KPISAMPLETYPE_', '')

            kpi_uuid = '{:s}-{:s}-{:s}-kpi_uuid'.format(DEVICE_EMU_UUID, endpoint_uuid, str(sample_type_id))

            monitoring_settings = {

                'kpi_id'        : {'kpi_id': {'uuid': kpi_uuid}},

                'kpi_descriptor': {

                    'kpi_description': 'Metric {:s} for Endpoint {:s} in Device {:s}'.format(

                SAMPLE_TYPE_NAME, ENDPOINT_UUID, DEVICE_EMU_UUID),

            'kpi_sample_type': SAMPLE_TYPE,

                        sample_type_name, endpoint_uuid, DEVICE_EMU_UUID),

                    'kpi_sample_type': sample_type_id,

                    'device_id': DEVICE_EMU_ID,

            'endpoint_id': endpoint_id(DEVICE_EMU_ID, ENDPOINT_UUID),

                    'endpoint_id': endpoint_id(DEVICE_EMU_ID, endpoint_uuid),

                },

                'sampling_duration_s': SAMPLING_DURATION_SEC,

                'sampling_interval_s': SAMPLING_INTERVAL_SEC,

            }

            MONITORING_SETTINGS_LIST.append(monitoring_settings)

            KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED[kpi_uuid] = 0



    NUM_SAMPLES_EXPECTED_PER_KPI = SAMPLING_DURATION_SEC / SAMPLING_INTERVAL_SEC

    NUM_SAMPLES_EXPECTED = len(MONITORING_SETTINGS_LIST) * NUM_SAMPLES_EXPECTED_PER_KPI



    # Start monitoring the device

    t_start_monitoring = datetime.timestamp(datetime.utcnow())

    device_client.MonitorDeviceKpi(MonitoringSettings(**MONITORING_SETTINGS))

    for monitoring_settings in MONITORING_SETTINGS_LIST:

        device_client.MonitorDeviceKpi(MonitoringSettings(**monitoring_settings))



    # wait to receive the expected number of samples

    # if takes more than 1.5 times the sampling duration, assume there is an error
@@ -357,17 +365,24 @@ def test_device_emulated_monitor( 
    LOGGER.info('received_samples = {:s}'.format(str(received_samples)))

    assert len(received_samples) == NUM_SAMPLES_EXPECTED

    for received_sample in received_samples:

        assert received_sample.kpi_id.kpi_id.uuid == KPI_UUID

        kpi_uuid = received_sample.kpi_id.kpi_id.uuid

        assert kpi_uuid in KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED

        assert isinstance(received_sample.timestamp, str)

        timestamp = float(received_sample.timestamp)

        assert timestamp > t_start_monitoring

        assert timestamp < t_end_monitoring

        assert received_sample.kpi_value.HasField('floatVal')

        assert isinstance(received_sample.kpi_value.floatVal, float)

        KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED[kpi_uuid] += 1



    LOGGER.info('KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED = {:s}'.format(str(KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED)))

    for kpi_uuid, num_samples_received in KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED.items():

        assert num_samples_received == NUM_SAMPLES_EXPECTED_PER_KPI



    # Unsubscribe monitoring

    for kpi_uuid in KPI_UUIDS__TO__NUM_SAMPLES_RECEIVED.keys():

        MONITORING_SETTINGS_UNSUBSCRIBE = {

        'kpi_id'             : {'kpi_id': {'uuid': KPI_UUID}},

            'kpi_id'             : {'kpi_id': {'uuid': kpi_uuid}},

            'sampling_duration_s': -1, # negative value in sampling_duration_s or in sampling_interval_s means unsibscribe

            'sampling_interval_s': -1, # kpi_id is mandatory to unsibscribe

        }