Commit b3cc3550 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/439-oc-driver-p2mp' into 'develop'

(CNIT) OpenConfig driver support for P2MP devices

Closes #439

See merge request !503
parents 1ba39bfe 06773408
Loading
Loading
Loading
Loading
+7 −1
Changes for src/device/service/DeviceServiceServicerImpl.py: 7 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -119,6 +119,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
        t3 = time.time()
        try:
            driver : _Driver = get_driver(self.driver_instance_cache, device)
            LOGGER.info("AddDevice in DeviceServiceServicerImpl started for device %s", device_uuid)

            t4 = time.time()

@@ -189,7 +190,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
                #    pass

                if 'new_optical_config' in new_optical_configs and 'opticalconfig' in new_optical_configs["new_optical_config"]:
                    LOGGER.info(f"set optical device config {new_optical_configs}")
                    #LOGGER.info(f"set optical device config {new_optical_configs}")
                    context_client.SetOpticalConfig(new_optical_configs["new_optical_config"]['opticalconfig'])

            device_id = context_client.SetDevice(device)
@@ -244,6 +245,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
            if len(new_sub_links) > 0:
                histogram_duration.labels(step='set_sub_links', **metrics_labels).observe(t12-t11)

            LOGGER.info("AddDevice in DeviceServiceServicerImpl device_id for device %s", device_id)
            return device_id
        finally:
            self.mutex_queues.signal_done(device_uuid)
@@ -277,6 +279,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):

            device_uuid = device.device_id.device_uuid.uuid
            driver : _Driver = get_driver(self.driver_instance_cache, device)
            LOGGER.info("ConfigureDevice in DeviceServiceServicerImpl started for device %s", device_uuid)
            if driver is None:
                msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid))
                raise OperationFailedException('ConfigureDevice', extra_details=msg)
@@ -363,6 +366,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):

            self.monitoring_loops.remove_device(device_uuid)
            self.driver_instance_cache.delete(device_uuid)
            LOGGER.info("DeleteDevice in DeviceServiceServicerImpl started for device %s", device_uuid)
            context_client.RemoveDevice(request)
            return Empty()
        finally:
@@ -382,6 +386,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
                raise NotFoundException('Device', device_uuid, extra_details='loading in DeleteDevice')

            driver : _Driver = get_driver(self.driver_instance_cache, device)
            LOGGER.info("GetInitialConfig in DeviceServiceServicerImpl started for device %s", device_uuid)
            if driver is None:
                msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid))
                raise OperationFailedException('GetInitialConfig', extra_details=msg)
@@ -423,6 +428,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer):
                raise NotFoundException('Device', device_uuid, extra_details='loading in DeleteDevice')

            driver : _Driver = get_driver(self.driver_instance_cache, device)
            LOGGER.info("MonitorDeviceKpi in DeviceServiceServicerImpl started for device %s", device_uuid)
            if driver is None:
                msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid))
                raise OperationFailedException('MonitorDeviceKpi', extra_details=msg)
+14 −3
Changes for src/device/service/OpenConfigServicer.py: 14 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ from context.client.ContextClient import ContextClient
from .driver_api._Driver import _Driver
from .driver_api.DriverInstanceCache import DriverInstanceCache, get_driver
from .monitoring.MonitoringLoops import MonitoringLoops
from .Tools import extract_resources
from .Tools import extract_resources, clear_frequency_tracking
from .Tools import check_no_endpoints

LOGGER = logging.getLogger(__name__)
@@ -81,6 +81,9 @@ class OpenConfigServicer(DeviceServiceServicer):
        resources : list[dict] = []
        is_all_good = True
        config = json.loads(request.config)
        #LOGGER.info(f"Openconfigservicer received config: {config}")
        LOGGER.info(f"Openconfigservicer track flow in device: {device_uuid}")
        print(f"Openconfigservicer track flow in device: {device_uuid}")
        results = None
        
        LOGGER.info(f"config from service {config}")
@@ -94,8 +97,10 @@ class OpenConfigServicer(DeviceServiceServicer):
            if device is None:
                raise NotFoundException('Device', device_uuid, extra_details='loading in ConfigureDevice')
            resources, conditions = extract_resources(config=config, device=device)
            LOGGER.info(f"resources  {resources}")
            LOGGER.info(f"conditions  {conditions}")
            LOGGER.info(f"this is the conditions: {conditions}")
            LOGGER.info(f"this is the resources: {resources}")
            print(f"this is the conditions: {conditions}")
            print(f"this is the resources: {resources}")
            driver : _Driver = get_driver(self.driver_instance_cache, device)
            results,new_config = driver.SetConfig(resources=resources,conditions=conditions)
        
@@ -174,6 +179,12 @@ class OpenConfigServicer(DeviceServiceServicer):
                raise NotFoundException('Device', device_uuid, extra_details='loading in ConfigureDevice')

            resources, conditions = extract_resources(config=config, device=device)
            clear_frequency_tracking()
            # Imported lazily: the oc_driver.templates package pulls in lxml/jinja2/paramiko
            # at import time, which aren't installed in components (e.g. monitoring) that
            # only import DeviceService for type access and never actually call DeleteConfig.
            from .drivers.oc_driver.templates.VPN.transponder import clear_frequency_tracking_transponder
            clear_frequency_tracking_transponder()

            driver : _Driver = get_driver(self.driver_instance_cache, device)
            results,config_delete = driver.DeleteConfig(resources=resources,conditions=conditions)
+431 −68

File changed.

Preview size limit exceeded, changes collapsed.

+14 −1
Changes for src/device/service/driver_api/DriverInstanceCache.py: 14 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -65,7 +65,13 @@ class DriverInstanceCache:
        with self._lock:
            device_driver = self._device_uuid__to__driver_instance.pop(device_uuid, None)
            if device_driver is None: return
            try:
                device_driver.Disconnect()
            except: # pylint: disable=bare-except
                # The driver is already out of the cache; a failed disconnect (e.g. the device
                # dropped the session) must not prevent the device from being deleted.
                msg = 'Error disconnecting Driver({:s}) from device. Removing it anyway...'
                LOGGER.exception(msg.format(device_uuid))

    def terminate(self) -> None:
        self._terminate.set()
@@ -111,4 +117,11 @@ def get_driver(driver_instance_cache : DriverInstanceCache, device : Device) ->
def preload_drivers(driver_instance_cache : DriverInstanceCache) -> None:
    context_client = ContextClient()
    devices = context_client.ListDevices(Empty())
    for device in devices.devices: get_driver(driver_instance_cache, device)
    for device in devices.devices:
        # An unreachable device must not take the whole Device component down at startup;
        # its driver is instantiated again on the next request that needs it.
        try:
            get_driver(driver_instance_cache, device)
        except: # pylint: disable=bare-except
            msg = 'Unable to pre-load driver for Device({:s}); skipping it'
            LOGGER.exception(msg.format(device.device_id.device_uuid.uuid))
+101 −25
Changes for src/device/service/drivers/oc_driver/OCDriver.py: 101 added lines, 25 removed lines.
Original line number Diff line number Diff line
@@ -35,8 +35,9 @@ from .RetryDecorator import retry
from context.client.ContextClient import ContextClient
from common.proto.context_pb2 import OpticalConfig
from .templates.discovery_tool.transponders import  transponder_values_extractor
from .templates.discovery_tool.open_roadm import openroadm_values_extractor
from .templates.discovery_tool.roadms import roadm_values_extractor

from .Tools import *


DEBUG_MODE = False
@@ -86,6 +87,13 @@ class NetconfSessionHandler:
       
    def connect(self):
        with self.__lock:
            # Reconnecting (e.g. from RETRY_DECORATOR after the device dropped the session):
            # release the previous session first so it does not linger on the device.
            if self.__manager is not None and self.__manager.connected:
                try:
                    self.__manager.close_session()
                except Exception:   # pylint: disable=broad-except
                    pass
            self.__manager = connect_ssh(
                host=self.__address, port=self.__port, username=self.__username, password=self.__password,
                device_params=self.__device_params, manager_params=self.__manager_params, nc_params=self.__nc_params,
@@ -97,7 +105,16 @@ class NetconfSessionHandler:
    def disconnect(self):
        if not self.__connected.is_set(): return
        with self.__lock:
            self.__connected.clear()
            # The device may already have dropped the session; closing it then raises
            # "Not connected to NETCONF server", which must not block device removal.
            if self.__manager is None or not self.__manager.connected: return
            try:
                self.__manager.close_session()
            except Exception:   # pylint: disable=broad-except
                logging.warning(
                    'Unable to close NETCONF session to {:s}:{:d}; dropping it'.format(self.__address, self.__port),
                    exc_info=True)

    @property
    def use_candidate(self): return self.__candidate_supported and not self.__force_running
@@ -270,6 +287,8 @@ def edit_config(
    return [results,new_config]         

class OCDriver(_Driver):


    def __init__(self, address : str, port : int,device_uuid=None, **settings) -> None:
        super().__init__(DRIVER_NAME, address, port, **settings)
        self.__logger = logging.getLogger('{:s}:[{:s}:{:s}]'.format(str(__name__), str(self.address), str(self.port)))
@@ -288,7 +307,16 @@ class OCDriver(_Driver):
        self.__netconf_handler = NetconfSessionHandler(self.address, self.port, **(self.settings))
        self.__type = self.settings.get("type","optical-transponder")
        self.__device_uuid = device_uuid


        self.Connect()
        logging.info("This is the settings %s", self.settings)
        # Fetch and log config immediately after connection
        try:
            logging.info("Device discovered. Fetching initial NETCONF config...")
        except Exception as e:
            logging.error(f"Failed to fetch initial config: {str(e)}")


    def Connect(self) -> bool:
        with self.__lock:
@@ -327,40 +355,88 @@ class OCDriver(_Driver):
                xml_data = self.__netconf_handler.get().data_xml
                
                if self.__type == "optical-transponder":
                    try:
                        extracted_values = transponder_values_extractor(
                        data_xml=xml_data, resource_keys=transponder_filter_fields, dic=config
                            data_xml=xml_data,
                            resource_keys=transponder_filter_fields, dic=config,
                            # dic={}
                        )
                        logging.info(f"Extracted values: {extracted_values}")

                        if len(extracted_values) >= 5:
                            interfaces, optical_channels_params, channel_namespace, endpoints, ports_result = extracted_values
                    oc_values["channels"         ] = optical_channels_params
                    oc_values["interfaces"       ] =interfaces   
                    oc_values["transceivers"     ] = transceivers
                    oc_values["channel_namespace"] = channel_namespace
                    oc_values["endpoints"        ] = endpoints
                    oc_values["ports"            ] = ports_result
                            oc_values.update({
                                "channels": optical_channels_params,
                                "interfaces": interfaces,
                                "channel_namespace": channel_namespace,
                                "endpoints": endpoints,
                                "ports": ports_result
                            })

                else:
                    extracted_values = roadm_values_extractor(data_xml=xml_data, resource_keys=[], dic=config)
                    ports_result = extracted_values[0]
                    oc_values['optical_bands' ] = extracted_values[1]
                    oc_values['media_channels'] = extracted_values[2]

                #results.append((resource_key, e)) # if validation fails, store the exception    
                    except Exception as e:
                        self.__logger.error(f"Error processing transponder data: {str(e)}", exc_info=True)
                        return []

            #///////////////////////// store optical configurtaion  ////////////////////////////////////////////////////////
                elif self.__type == "openroadm":
                    try:
                        extracted_values = openroadm_values_extractor(
                            data_xml=xml_data,
                            resource_keys=[],
                            dic=oc_values
                        )
                        if len(extracted_values) >= 2:
                            oc_values['interfaces'] = extracted_values[0]['interfaces']
                            ports_result = extracted_values[1]
                            oc_values['ports'] = ports_result
                    except Exception as e:
                        self.__logger.error(f"Error processing openroadm data: {str(e)}", exc_info=True)
                        return []

                else:  # optical-roadm
                    try:
                        extracted_values = roadm_values_extractor(
                            data_xml=xml_data,
                            resource_keys=[],
                            dic={}
                        )
                        if len(extracted_values) >= 3:
                            ports_result = extracted_values[0]
                            oc_values.update({
                                'ports': ports_result,
                                'optical_bands': extracted_values[1],
                                'media_channels': extracted_values[2]
                            })
                    except Exception as e:
                        self.__logger.error(f"Error processing roadm data: {str(e)}", exc_info=True)
                        return []

                # 4. Create OpticalConfig protobuf
                opticalConfig = OpticalConfig()
                try:
                    opticalConfig.config = json.dumps(oc_values)
                if self.__device_uuid is not None:
                    if self.__device_uuid:
                        opticalConfig.device_id.device_uuid.uuid = self.__device_uuid
                results.append((f"/opticalconfigs/opticalconfig/{self.__device_uuid}",{"opticalconfig":opticalConfig}))
                # context_client.connect()    
                # config_id=context_client.SetOpticalConfig(opticalConfig)
                # context_client.close()
            except Exception as e: # pylint: disable=broad-except
                MSG = 'Exception retrieving {:s}'
                self.__logger.info("error from getConfig %s",e)
                self.__logger.exception(MSG.format(e))

        if len(ports_result) > 0: results.extend(ports_result)
                    # 5. Add to results
                    config_path = f"/opticalconfigs/opticalconfig/{self.__device_uuid}"
                    results.append((config_path, {"opticalconfig": opticalConfig}))

                    # 6. Add port results if any
                    if 'ports' in oc_values and oc_values['ports']:
                        results.extend(oc_values['ports'])

                    #self.__logger.info(f"Successfully processed config for device {self.__device_uuid}")
                    #self.__logger.debug(f"Final Optical Config: {opticalConfig}")

                except Exception as e:
                    self.__logger.error(f"Error creating OpticalConfig: {str(e)}", exc_info=True)
                    return []

            except Exception as e:
                self.__logger.error(f"NETCONF operation failed: {str(e)}", exc_info=True)
                return []

        return results

    @metered_subclass_method(METRICS_POOL)
Loading