Commit 5f31fb8b authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Device component: 

- Add driver name attribute to drivers
- Improve instantiation of per-driver MetricsPool
- Added detailed metrics for AddDevice RPC method
parent d2dc573e

src/device/service/DeviceServiceServicerImpl.py

+68 −9
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ LOGGER = logging.getLogger(__name__) 
METRICS_POOL = MetricsPool('Device', 'RPC')



METRICS_POOL_DETAILS = MetricsPool('Device', 'exec_details', labels={

    'step_name': '',

    'driver': '', 'operation': '', 'step': '',

})



class DeviceServiceServicerImpl(DeviceServiceServicer):
@@ -51,11 +51,15 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): 


    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def AddDevice(self, request : Device, context : grpc.ServicerContext) -> DeviceId:

        t0 = time.time()



        device_uuid = request.device_id.device_uuid.uuid



        connection_config_rules = check_connect_rules(request.device_config)

        check_no_endpoints(request.device_endpoints)



        t1 = time.time()



        context_client = ContextClient()

        device = get_device(context_client, device_uuid, rw_copy=True)

        if device is None:
@@ -73,10 +77,15 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): 
        # update device_uuid to honor UUID provided by Context

        device_uuid = device.device_id.device_uuid.uuid



        t2 = time.time()



        self.mutex_queues.wait_my_turn(device_uuid)

        t3 = time.time()

        try:

            driver : _Driver = get_driver(self.driver_instance_cache, device)



            t4 = time.time()



            errors = []



            # Sub-devices and sub-links are exposed by intermediate controllers or represent mgmt links.
@@ -86,13 +95,23 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): 
            new_sub_links : Dict[str, Link] = dict()



            if len(device.device_endpoints) == 0:

                t5 = time.time()

                # created from request, populate endpoints using driver

                errors.extend(populate_endpoints(

                    device, driver, self.monitoring_loops, new_sub_devices, new_sub_links))

                t6 = time.time()

                t_pop_endpoints = t6 - t5

            else:

                t_pop_endpoints = None



            if len(device.device_config.config_rules) == len(connection_config_rules):

                # created from request, populate config rules using driver

                t7 = time.time()

                errors.extend(populate_config_rules(device, driver))

                t8 = time.time()

                t_pop_config_rules = t8 - t7

            else:

                t_pop_config_rules = None



            # TODO: populate components
@@ -100,22 +119,60 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): 
                for error in errors: LOGGER.error(error)

                raise OperationFailedException('AddDevice', extra_details=errors)



            t9 = time.time()



            device.device_operational_status = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_DISABLED

            device_id = context_client.SetDevice(device)



            t10 = time.time()



            for sub_device in new_sub_devices.values():

                context_client.SetDevice(sub_device)



            t11 = time.time()



            for sub_links in new_sub_links.values():

                context_client.SetLink(sub_links)



            t12 = time.time()



            # Update endpoint monitoring resources with UUIDs

            device_with_uuids = get_device(

                context_client, device_id.device_uuid.uuid, rw_copy=False, include_endpoints=True,

                include_components=False, include_config_rules=False)

            populate_endpoint_monitoring_resources(device_with_uuids, self.monitoring_loops)



            t13 = time.time()



            context_client.close()



            t14 = time.time()



            metrics_labels = dict(driver=driver.name, operation='add_device')



            histogram_duration : Histogram = METRICS_POOL_DETAILS.get_or_create(

                'AddDevice', MetricTypeEnum.HISTOGRAM_DURATION)

            histogram_duration.labels(step='total'              , **metrics_labels).observe(t14-t0)

            histogram_duration.labels(step='execution'          , **metrics_labels).observe(t14-t3)

            histogram_duration.labels(step='endpoint_checks'    , **metrics_labels).observe(t1-t0)

            histogram_duration.labels(step='get_device'         , **metrics_labels).observe(t2-t1)

            histogram_duration.labels(step='wait_queue'         , **metrics_labels).observe(t3-t2)

            histogram_duration.labels(step='get_driver'         , **metrics_labels).observe(t4-t3)

            histogram_duration.labels(step='set_device'         , **metrics_labels).observe(t10-t9)

            histogram_duration.labels(step='populate_monit_rsrc', **metrics_labels).observe(t13-t12)



            if t_pop_endpoints is not None:

                histogram_duration.labels(step='populate_endpoints', **metrics_labels).observe(t_pop_endpoints)



            if t_pop_config_rules is not None:

                histogram_duration.labels(step='populate_config_rules', **metrics_labels).observe(t_pop_config_rules)



            if len(new_sub_devices) > 0:

                histogram_duration.labels(step='set_sub_devices', **metrics_labels).observe(t11-t10)



            if len(new_sub_links) > 0:

                histogram_duration.labels(step='set_sub_links', **metrics_labels).observe(t12-t11)



            return device_id

        finally:

            self.mutex_queues.signal_done(device_uuid)
@@ -195,16 +252,18 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): 


            t9 = time.time()



            metrics_labels = dict(driver=driver.name, operation='configure_device')



            histogram_duration : Histogram = METRICS_POOL_DETAILS.get_or_create(

                'ConfigureDevice', MetricTypeEnum.HISTOGRAM_DURATION)

            histogram_duration.labels(step_name='total'            ).observe(t9-t0)

            histogram_duration.labels(step_name='wait_queue'       ).observe(t1-t0)

            histogram_duration.labels(step_name='execution'        ).observe(t9-t1)

            histogram_duration.labels(step_name='get_device'       ).observe(t3-t2)

            histogram_duration.labels(step_name='split_rules'      ).observe(t5-t4)

            histogram_duration.labels(step_name='configure_rules'  ).observe(t6-t5)

            histogram_duration.labels(step_name='deconfigure_rules').observe(t7-t6)

            histogram_duration.labels(step_name='set_device'       ).observe(t9-t8)

            histogram_duration.labels(step='total'            , **metrics_labels).observe(t9-t0)

            histogram_duration.labels(step='wait_queue'       , **metrics_labels).observe(t1-t0)

            histogram_duration.labels(step='execution'        , **metrics_labels).observe(t9-t1)

            histogram_duration.labels(step='get_device'       , **metrics_labels).observe(t3-t2)

            histogram_duration.labels(step='split_rules'      , **metrics_labels).observe(t5-t4)

            histogram_duration.labels(step='configure_rules'  , **metrics_labels).observe(t6-t5)

            histogram_duration.labels(step='deconfigure_rules', **metrics_labels).observe(t7-t6)

            histogram_duration.labels(step='set_device'       , **metrics_labels).observe(t9-t8)



            return device_id

        finally:

src/device/service/driver_api/_Driver.py

+17 −2
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ RESOURCE_ACL = '__acl__' 




class _Driver:

    def __init__(self, address: str, port: int, **settings) -> None:

    def __init__(self, name : str, address: str, port: int, **settings) -> None:

        """ Initialize Driver.

            Parameters:

                address : str
@@ -37,7 +37,22 @@ class _Driver: 
                **settings

                    Extra settings required by the driver.

        """

        raise NotImplementedError()

        self._name = name

        self._address = address

        self._port = port

        self._settings = settings



    @property

    def name(self): return self._name



    @property

    def address(self): return self._address



    @property

    def port(self): return self._port



    @property

    def settings(self): return self._settings



    def Connect(self) -> bool:

        """ Connect to the Device.

src/device/service/drivers/emulated/EmulatedDriver.py

+5 −3
Original line number Diff line number Diff line
@@ -31,16 +31,18 @@ LOGGER = logging.getLogger(__name__) 


RE_GET_ENDPOINT_FROM_INTERFACE = re.compile(r'^\/interface\[([^\]]+)\].*')



METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'emulated'})

DRIVER_NAME = 'emulated'

METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME})



class EmulatedDriver(_Driver):

    def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called

    def __init__(self, address : str, port : int, **settings) -> None:

        super().__init__(DRIVER_NAME, address, port, **settings)

        self.__lock = threading.Lock()

        self.__initial = TreeNode('.')

        self.__running = TreeNode('.')

        self.__subscriptions = TreeNode('.')



        endpoints = settings.get('endpoints', [])

        endpoints = self.settings.get('endpoints', [])

        endpoint_resources = []

        for endpoint in endpoints:

            endpoint_resource = compose_resource_endpoint(endpoint)

src/device/service/drivers/ietf_l2vpn/IetfL2VpnDriver.py

+9 −7
Original line number Diff line number Diff line
@@ -39,21 +39,23 @@ ALL_RESOURCE_KEYS = [ 


SERVICE_TYPE = 'ELINE'



METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'ietf_l2vpn'})

DRIVER_NAME = 'ietf_l2vpn'

METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME})



class IetfL2VpnDriver(_Driver):

    def __init__(self, address: str, port: int, **settings) -> None:    # pylint: disable=super-init-not-called

    def __init__(self, address: str, port: int, **settings) -> None:

        super().__init__(DRIVER_NAME, address, port, **settings)

        self.__lock = threading.Lock()

        self.__started = threading.Event()

        self.__terminate = threading.Event()

        username = settings.get('username')

        password = settings.get('password')

        scheme = settings.get('scheme', 'http')

        wim = {'wim_url': '{:s}://{:s}:{:d}'.format(scheme, address, int(port))}

        username = self.settings.get('username')

        password = self.settings.get('password')

        scheme = self.settings.get('scheme', 'http')

        wim = {'wim_url': '{:s}://{:s}:{:d}'.format(scheme, self.address, int(self.port))}

        wim_account = {'user': username, 'password': password}

        # Mapping updated dynamically with each request

        config = {'mapping_not_needed': False, 'service_endpoint_mapping': []}

        self.dac = TfsDebugApiClient(address, int(port), scheme=scheme, username=username, password=password)

        self.dac = TfsDebugApiClient(self.address, int(self.port), scheme=scheme, username=username, password=password)

        self.wim = WimconnectorIETFL2VPN(wim, wim_account, config=config)

        self.conn_info = {} # internal database emulating OSM storage provided to WIM Connectors

src/device/service/drivers/microwave/IETFApiDriver.py

+10 −8
Original line number Diff line number Diff line
@@ -23,20 +23,22 @@ from .Tools import create_connectivity_service, find_key, config_getter, delete_ 


LOGGER = logging.getLogger(__name__)



METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'microwave'})

DRIVER_NAME = 'microwave'

METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME})



class IETFApiDriver(_Driver):

    def __init__(self, address: str, port: int, **settings) -> None:    # pylint: disable=super-init-not-called

    def __init__(self, address: str, port: int, **settings) -> None:

        super().__init__(DRIVER_NAME, address, port, **settings)

        self.__lock = threading.Lock()

        self.__started = threading.Event()

        self.__terminate = threading.Event()

        username = settings.get('username')

        password = settings.get('password')

        username = self.settings.get('username')

        password = self.settings.get('password')

        self.__auth = HTTPBasicAuth(username, password) if username is not None and password is not None else None

        scheme = settings.get('scheme', 'http')

        self.__ietf_root = '{:s}://{:s}:{:d}'.format(scheme, address, int(port))

        self.__timeout = int(settings.get('timeout', 120))

        self.__node_ids = set(settings.get('node_ids', []))

        scheme = self.settings.get('scheme', 'http')

        self.__ietf_root = '{:s}://{:s}:{:d}'.format(scheme, self.address, int(self.port))

        self.__timeout = int(self.settings.get('timeout', 120))

        self.__node_ids = set(self.settings.get('node_ids', []))



    def Connect(self) -> bool:

        url = self.__ietf_root + '/nmswebs/restconf/data/ietf-network:networks'