Commit 07276892 authored by Pablo Armingol's avatar Pablo Armingol
Browse files

Merge branch 'feat/device-perf-eval' of... 

Merge branch 'feat/device-perf-eval' of https://labs.etsi.org/rep/tfs/controller into perf/tid-openconfig
parents e328939c 42de2fbe

src/device/service/drivers/microwave/IETFApiDriver.py

+15 −5
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
# limitations under the License.



import logging, requests, threading

from requests.auth import HTTPBasicAuth

from typing import Any, Iterator, List, Optional, Tuple, Union

from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method

from common.type_checkers.Checkers import chk_string, chk_type
@@ -29,15 +30,20 @@ class IETFApiDriver(_Driver): 
        self.__lock = threading.Lock()

        self.__started = threading.Event()

        self.__terminate = threading.Event()

        self.__ietf_root = 'https://' + address + ':' + str(port)

        username = settings.get('username')

        password = settings.get('password')

        self.__auth = HTTPBasicAuth(username, password) if username is not None and password is not None else None

        scheme = settings.get('scheme', 'http')

        self.__ietf_root = '{:s}://{:s}:{:d}'.format(scheme, address, int(port))

        self.__timeout = int(settings.get('timeout', 120))

        self.__node_ids = set(settings.get('node_ids', []))



    def Connect(self) -> bool:

        url = self.__ietf_root + '/nmswebs/restconf/data/ietf-network:networks'

        with self.__lock:

            if self.__started.is_set(): return True

            try:

                requests.get(url, timeout=self.__timeout, verify=False)

                requests.get(url, timeout=self.__timeout, verify=False, auth=self.__auth)

            except requests.exceptions.Timeout:

                LOGGER.exception('Timeout connecting {:s}'.format(str(self.__ietf_root)))

                return False
@@ -67,7 +73,9 @@ class IETFApiDriver(_Driver): 
            for i, resource_key in enumerate(resource_keys):

                str_resource_name = 'resource_key[#{:d}]'.format(i)

                chk_string(str_resource_name, resource_key, allow_empty=False)

                results.extend(config_getter(self.__ietf_root, resource_key, self.__timeout))

                results.extend(config_getter(

                    self.__ietf_root, resource_key, timeout=self.__timeout, auth=self.__auth,

                    node_ids=self.__node_ids))

        return results



    @metered_subclass_method(METRICS_POOL)
@@ -87,7 +95,8 @@ class IETFApiDriver(_Driver): 
                uuid = find_key(resource, 'uuid')



                data = create_connectivity_service(

                    self.__ietf_root, self.__timeout, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id)

                    self.__ietf_root, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id,

                    timeout=self.__timeout, auth=self.__auth)

                results.extend(data)

        return results
@@ -99,7 +108,8 @@ class IETFApiDriver(_Driver): 
            for resource in resources:

                LOGGER.info('resource = {:s}'.format(str(resource)))

                uuid = find_key(resource, 'uuid')

                results.extend(delete_connectivity_service(self.__ietf_root, self.__timeout, uuid))

                results.extend(delete_connectivity_service(

                    self.__ietf_root, uuid, timeout=self.__timeout, auth=self.__auth))

        return results



    @metered_subclass_method(METRICS_POOL)

src/device/service/drivers/microwave/Tools.py

+46 −36
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@ 
# limitations under the License.



import json, logging, requests

from requests.auth import HTTPBasicAuth

from typing import Optional, Set

from device.service.driver_api._Driver import RESOURCE_ENDPOINTS



LOGGER = logging.getLogger(__name__)
@@ -28,6 +30,8 @@ def find_key(resource, key): 
    return json.loads(resource[1])[key]



# this function exports only the endpoints which are not already involved in a microwave physical link

# TODO: improvement: create a Set[Tuple[node_id:str, tp_id:str]] containing the endpoints involved in links

# TODO: exportable endpoints are those not in this set. That will prevent looping through links for every endpoint

def is_exportable_endpoint(node, termination_point_id, links):

    # for each link we check if the endpoint (termination_point_id) is already used by an existing link

    for link in links:
@@ -39,7 +43,10 @@ def is_exportable_endpoint(node, termination_point_id, links): 
            return False

    return True



def config_getter(root_url, resource_key, timeout):

def config_getter(

    root_url : str, resource_key : str, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None,

    node_ids : Set[str] = set()

):

    # getting endpoints

    network_name = 'SIAE-ETH-TOPOLOGY'

    FIELDS = ''.join([
@@ -51,51 +58,53 @@ def config_getter(root_url, resource_key, timeout): 
    url = URL_TEMPLATE.format(root_url, network_name, FIELDS)



    result = []

    if resource_key == RESOURCE_ENDPOINTS:

        # getting existing endpoints

        try:

        response = requests.get(url, timeout=timeout, verify=False)

    except requests.exceptions.Timeout:

        LOGGER.exception('Timeout connecting {:s}'.format(url))

    except Exception as e:  # pylint: disable=broad-except

        LOGGER.exception('Exception retrieving {:s}'.format(resource_key))

        result.append((resource_key, e))

    else:

            response = requests.get(url, timeout=timeout, verify=False, auth=auth)

            context = json.loads(response.content)

        if resource_key == RESOURCE_ENDPOINTS:

            network_instance = context.get('ietf-network:network', {})

            links = network_instance.get('ietf-network-topology:link', [])

            for sip in network_instance['node']:

                tp = sip['ietf-network-topology:termination-point']

                node_id = sip['node-id']

                for te in tp:

                    tp_id = te['tp-id']

            for node in network_instance['node']:

                node_id = node['node-id']

                if len(node_ids) > 0 and node_id not in node_ids: continue

                tp_list = node['ietf-network-topology:termination-point']

                for tp in tp_list:

                    tp_id = tp['tp-id']

                    if not is_exportable_endpoint(node_id, tp_id, links): continue

                    resource_key = '/endpoints/endpoint[{:s}:{:s}]'.format(node_id,tp_id)

                    resource_value = {'uuid': tp_id, 'type': te['ietf-te-topology:te']['name']}

                    tp_uuid = '{:s}:{:s}'.format(node_id,tp_id)

                    resource_key = '/endpoints/endpoint[{:s}]'.format(tp_uuid)

                    resource_value = {'uuid': tp_uuid, 'type': tp['ietf-te-topology:te']['name']}

                    result.append((resource_key, resource_value))



    # getting created services

    url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)

    try:

        response = requests.get(url, timeout=timeout, verify=False)

        except requests.exceptions.Timeout:

            LOGGER.exception('Timeout connecting {:s}'.format(url))

        except Exception as e:  # pylint: disable=broad-except

        LOGGER.exception('Exception retrieving {:s}'.format(resource_key))

            LOGGER.exception('Exception retrieving/parsing endpoints for {:s}'.format(resource_key))

            result.append((resource_key, e))

    else:

        # getting created services

        url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)

        try:

            response = requests.get(url, timeout=timeout, verify=False, auth=auth)

            context = json.loads(response.content)

        if resource_key == RESOURCE_ENDPOINTS:

            etht_service = context.get('ietf-eth-tran-service:etht-svc', {})

            service_instances = etht_service.get('etht-svc-instances', [])

            for service in service_instances:

                service_name = service['etht-svc-name']

                resource_key = '/services/service[{:s}]'.format(service_name)

                resource_value = {'uuid': service_name, 'type': service['etht-svc-type']}

                result.append((resource_key, resource_value))

                result.append((resource_key, service))

        except requests.exceptions.Timeout:

            LOGGER.exception('Timeout connecting {:s}'.format(url))

        except Exception as e:  # pylint: disable=broad-except

            LOGGER.exception('Exception retrieving/parsing services for {:s}'.format(resource_key))

            result.append((resource_key, e))



    return result



def create_connectivity_service(

    root_url, timeout, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id):

    root_url, uuid, node_id_src, tp_id_src, node_id_dst, tp_id_dst, vlan_id,

    auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None

):



    url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc'.format(root_url)

    headers = {'content-type': 'application/json'}
@@ -128,7 +137,8 @@ def create_connectivity_service( 
    results = []

    try:

        LOGGER.info('Connectivity service {:s}: {:s}'.format(str(uuid), str(data)))

        response = requests.post(url=url, data=json.dumps(data), timeout=timeout, headers=headers, verify=False)

        response = requests.post(

            url=url, data=json.dumps(data), timeout=timeout, headers=headers, verify=False, auth=auth)

        LOGGER.info('Microwave Driver response: {:s}'.format(str(response)))

    except Exception as e:  # pylint: disable=broad-except

        LOGGER.exception('Exception creating ConnectivityService(uuid={:s}, data={:s})'.format(str(uuid), str(data)))
@@ -140,12 +150,12 @@ def create_connectivity_service( 
        results.append(response.status_code in HTTP_OK_CODES)

    return results



def delete_connectivity_service(root_url, timeout, uuid):

def delete_connectivity_service(root_url, uuid, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None):

    url = '{:s}/nmswebs/restconf/data/ietf-eth-tran-service:etht-svc/etht-svc-instances={:s}'

    url = url.format(root_url, uuid)

    results = []

    try:

        response = requests.delete(url=url, timeout=timeout, verify=False)

        response = requests.delete(url=url, timeout=timeout, verify=False, auth=auth)

    except Exception as e:  # pylint: disable=broad-except

        LOGGER.exception('Exception deleting ConnectivityService(uuid={:s})'.format(str(uuid)))

        results.append(e)

src/service/service/service_handlers/microwave/MicrowaveServiceHandler.py

+5 −3
Original line number Diff line number Diff line
@@ -81,7 +81,7 @@ class MicrowaveServiceHandler(_ServiceHandler): 
        

            device_uuid = endpoints[0][0]

            device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid)))

            json_config_rule = json_config_rule_set('/service[{:s}]'.format(service_uuid), {

            json_config_rule = json_config_rule_set('/services/service[{:s}]'.format(service_uuid), {

                'uuid'       : service_uuid,

                'node_id_src': node_id_src,

                'tp_id_src'  : tp_id_src,
@@ -111,11 +111,13 @@ class MicrowaveServiceHandler(_ServiceHandler): 
        results = []

        try:

            chk_type('endpoints', endpoints, list)

            if len(endpoints) != 2: raise Exception('len(endpoints) != 2')

            if len(endpoints) < 1: raise Exception('len(endpoints) < 1')



            device_uuid = endpoints[0][0]

            device = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid)))

            json_config_rule = json_config_rule_delete('/service[{:s}]'.format(service_uuid), {'uuid': service_uuid})

            json_config_rule = json_config_rule_delete('/services/service[{:s}]'.format(service_uuid), {

                'uuid': service_uuid

            })

            del device.device_config.config_rules[:]

            device.device_config.config_rules.append(ConfigRule(**json_config_rule))

            self.__task_executor.configure_device(device)

src/tests/tools/load_gen/Constants.py

+1 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ class RequestType(Enum): 
    SERVICE_L2NM = 'svc-l2nm'

    SERVICE_L3NM = 'svc-l3nm'

    SERVICE_TAPI = 'svc-tapi'

    SERVICE_MW   = 'svc-mw'

    SLICE_L2NM   = 'slc-l2nm'

    SLICE_L3NM   = 'slc-l3nm'

src/tests/tools/load_gen/RequestGenerator.py

+36 −13
Original line number Diff line number Diff line
@@ -109,7 +109,8 @@ class RequestGenerator: 
            LOGGER.info('[dump_state] used_device_endpoints = {:s}'.format(json.dumps(self._used_device_endpoints)))



    def _use_device_endpoint(

        self, service_uuid : str, endpoint_types : Optional[Set[str]] = None, exclude_device_uuids : Set[str] = set()

        self, service_uuid : str, request_type : RequestType, endpoint_types : Optional[Set[str]] = None,

        exclude_device_uuids : Set[str] = set(), exclude_endpoint_uuids : Set[Tuple[str, str]] = set(), 

    ) -> Optional[Tuple[str, str]]:

        with self._lock:

            compatible_endpoints : Set[Tuple[str, str]] = set()
@@ -118,9 +119,14 @@ class RequestGenerator: 
            if endpoint_types is None:

                # allow all

                elegible_device_endpoints : Dict[str, Set[str]] = {

                    device_uuid:device_endpoint_uuids

                    device_uuid:[

                        endpoint_uuid for endpoint_uuid in device_endpoint_uuids

                        if (len(exclude_endpoint_uuids) == 0) or \

                            ((device_uuid,endpoint_uuid) not in exclude_endpoint_uuids)

                    ]

                    for device_uuid,device_endpoint_uuids in self._available_device_endpoints.items()

                    if device_uuid not in exclude_device_uuids and len(device_endpoint_uuids) > 0

                    if (device_uuid not in exclude_device_uuids) and \

                        (len(device_endpoint_uuids) > 0)

                }

            else:

                # allow only compatible endpoints
@@ -132,6 +138,7 @@ class RequestGenerator: 
                    if device_uuid in exclude_device_uuids or len(device_endpoint_uuids) == 0: continue

                    for endpoint_uuid in device_endpoint_uuids:

                        endpoint_key = (device_uuid,endpoint_uuid)

                        if endpoint_key in exclude_endpoint_uuids: continue

                        if endpoint_key not in compatible_endpoints: continue

                        elegible_device_endpoints.setdefault(device_uuid, set()).add(endpoint_uuid)
@@ -145,16 +152,19 @@ class RequestGenerator: 
                    'compatible_endpoints={:s}'.format(str(compatible_endpoints)),

                ]))

                return None



            device_uuid = random.choice(list(elegible_device_endpoints.keys()))

            device_endpoint_uuids = elegible_device_endpoints.get(device_uuid)

            endpoint_uuid = random.choice(list(device_endpoint_uuids))

            if request_type not in {RequestType.SERVICE_MW}:

                # reserve the resources

                self._available_device_endpoints.setdefault(device_uuid, set()).discard(endpoint_uuid)

                self._used_device_endpoints.setdefault(device_uuid, dict())[endpoint_uuid] = service_uuid

            return device_uuid, endpoint_uuid



    def _release_device_endpoint(self, device_uuid : str, endpoint_uuid : str) -> None:

        with self._lock:

            self._used_device_endpoints.setdefault(device_uuid, set()).pop(endpoint_uuid, None)

            self._used_device_endpoints.setdefault(device_uuid, dict()).pop(endpoint_uuid, None)

            self._available_device_endpoints.setdefault(device_uuid, set()).add(endpoint_uuid)



    def compose_request(self) -> Optional[Dict]:
@@ -168,7 +178,9 @@ class RequestGenerator: 
        # choose request type

        request_type = random.choice(self._parameters.request_types)



        if request_type in {RequestType.SERVICE_L2NM, RequestType.SERVICE_L3NM, RequestType.SERVICE_TAPI}:

        if request_type in {

            RequestType.SERVICE_L2NM, RequestType.SERVICE_L3NM, RequestType.SERVICE_TAPI, RequestType.SERVICE_MW

        }:

            return self._compose_service(num_request, request_uuid, request_type)

        elif request_type in {RequestType.SLICE_L2NM, RequestType.SLICE_L3NM}:

            return self._compose_slice(num_request, request_uuid, request_type)
@@ -176,7 +188,7 @@ class RequestGenerator: 
    def _compose_service(self, num_request : int, request_uuid : str, request_type : str) -> Optional[Dict]:

        # choose source endpoint

        src_endpoint_types = set(ENDPOINT_COMPATIBILITY.keys()) if request_type in {RequestType.SERVICE_TAPI} else None

        src = self._use_device_endpoint(request_uuid, endpoint_types=src_endpoint_types)

        src = self._use_device_endpoint(request_uuid, request_type, endpoint_types=src_endpoint_types)

        if src is None:

            LOGGER.warning('>> No source endpoint is available')

            return None
@@ -188,11 +200,12 @@ class RequestGenerator: 
        dst_endpoint_types = {dst_endpoint_type} if request_type in {RequestType.SERVICE_TAPI} else None



        # identify excluded destination devices

        exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI} else {src_device_uuid}

        exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI, RequestType.SERVICE_MW} else {src_device_uuid}



        # choose feasible destination endpoint

        dst = self._use_device_endpoint(

            request_uuid, endpoint_types=dst_endpoint_types, exclude_device_uuids=exclude_device_uuids)

            request_uuid, request_type, endpoint_types=dst_endpoint_types, exclude_device_uuids=exclude_device_uuids,

            exclude_endpoint_uuids={src})

        

        # if destination endpoint not found, release source, and terminate current service generation

        if dst is None:
@@ -310,19 +323,29 @@ class RequestGenerator: 
            return json_service_tapi_planned(

                request_uuid, endpoint_ids=endpoint_ids, constraints=[], config_rules=config_rules)



        elif request_type == RequestType.SERVICE_MW:

            vlan_id = 1000 + num_request % 1000

            config_rules = [

                json_config_rule_set('/settings', {

                    'vlan_id': vlan_id,

                }),

            ]

            return json_service_l2nm_planned(

                request_uuid, endpoint_ids=endpoint_ids, constraints=[], config_rules=config_rules)



    def _compose_slice(self, num_request : int, request_uuid : str, request_type : str) -> Optional[Dict]:

        # choose source endpoint

        src = self._use_device_endpoint(request_uuid)

        src = self._use_device_endpoint(request_uuid, request_type)

        if src is None:

            LOGGER.warning('>> No source endpoint is available')

            return None

        src_device_uuid,src_endpoint_uuid = src



        # identify excluded destination devices

        exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI} else {src_device_uuid}

        exclude_device_uuids = {} if request_type in {RequestType.SERVICE_TAPI, RequestType.SERVICE_MW} else {src_device_uuid}



        # choose feasible destination endpoint

        dst = self._use_device_endpoint(request_uuid, exclude_device_uuids=exclude_device_uuids)

        dst = self._use_device_endpoint(request_uuid, request_type, exclude_device_uuids=exclude_device_uuids)

        

        # if destination endpoint not found, release source, and terminate current service generation

        if dst is None: