Commit 5cc92a6f authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/device-monitoring' into 'develop' 

Final arrangements in Device monitoring

See merge request teraflow-h2020/controller!49
parents 492e852b f3e8ac33

src/context/service/database/EndPointModel.py

+33 −3
Original line number Diff line number Diff line 
import logging

from typing import Dict

from typing import Dict, List

from common.orm.Database import Database

from common.orm.backend.Tools import key_to_str

from common.orm.fields.EnumeratedField import EnumeratedField

from common.orm.fields.ForeignKeyField import ForeignKeyField

from common.orm.fields.PrimaryKeyField import PrimaryKeyField

from common.orm.fields.StringField import StringField

from common.orm.model.Model import Model

from .DeviceModel import DeviceModel

from .KpiSampleType import ORM_KpiSampleTypeEnum, grpc_to_enum__kpi_sample_type

from .TopologyModel import TopologyModel



LOGGER = logging.getLogger(__name__)
@@ -26,8 +30,34 @@ class EndPointModel(Model): 
            result['topology_id'] = TopologyModel(self.database, self.topology_fk).dump_id()

        return result



    def dump(self) -> Dict:

        return {

    def dump_kpi_sample_types(self) -> List[int]:

        db_kpi_sample_type_pks = self.references(KpiSampleTypeModel)

        return [KpiSampleTypeModel(self.database, pk).dump() for pk,_ in db_kpi_sample_type_pks]



    def dump(   # pylint: disable=arguments-differ

            self, include_kpi_sample_types=True

        ) -> Dict:

        result = {

            'endpoint_id': self.dump_id(),

            'endpoint_type': self.endpoint_type,

        }

        if include_kpi_sample_types: result['kpi_sample_types'] = self.dump_kpi_sample_types()

        return result



class KpiSampleTypeModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()

    endpoint_fk = ForeignKeyField(EndPointModel)

    kpi_sample_type = EnumeratedField(ORM_KpiSampleTypeEnum, required=True)



    def dump(self) -> Dict:

        return self.kpi_sample_type.value



def set_kpi_sample_types(database : Database, db_endpoint : EndPointModel, grpc_endpoint_kpi_sample_types):

    db_endpoint_pk = db_endpoint.pk

    for kpi_sample_type in grpc_endpoint_kpi_sample_types:

        orm_kpi_sample_type = grpc_to_enum__kpi_sample_type(kpi_sample_type)

        str_endpoint_kpi_sample_type_key = key_to_str([db_endpoint_pk, orm_kpi_sample_type.name])

        db_endpoint_kpi_sample_type = KpiSampleTypeModel(database, str_endpoint_kpi_sample_type_key)

        db_endpoint_kpi_sample_type.endpoint_fk = db_endpoint

        db_endpoint_kpi_sample_type.kpi_sample_type = orm_kpi_sample_type

        db_endpoint_kpi_sample_type.save()

src/context/service/database/KpiSampleType.py

0 → 100644
+14 −0
Original line number Diff line number Diff line 
import functools

from enum import Enum

from context.proto.kpi_sample_types_pb2 import KpiSampleType

from .Tools import grpc_to_enum



class ORM_KpiSampleTypeEnum(Enum):

    UNKNOWN             = KpiSampleType.KPISAMPLETYPE_UNKNOWN

    PACKETS_TRANSMITTED = KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED

    PACKETS_RECEIVED    = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED

    BYTES_TRANSMITTED   = KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED

    BYTES_RECEIVED      = KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED



grpc_to_enum__kpi_sample_type = functools.partial(

    grpc_to_enum, KpiSampleType, ORM_KpiSampleTypeEnum)

src/context/service/grpc_server/ContextServiceServicerImpl.py

+8 −3
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ from context.service.database.ConstraintModel import ConstraintModel, Constraint 
from context.service.database.ContextModel import ContextModel

from context.service.database.DeviceModel import (

    DeviceModel, DriverModel, grpc_to_enum__device_operational_status, set_drivers)

from context.service.database.EndPointModel import EndPointModel

from context.service.database.EndPointModel import EndPointModel, KpiSampleTypeModel, set_kpi_sample_types

from context.service.database.Events import notify_event

from context.service.database.LinkModel import LinkModel

from context.service.database.RelationModels import (
@@ -279,7 +279,9 @@ class ContextServiceServicerImpl(ContextServiceServicer): 


            result : Tuple[EndPointModel, bool] = update_or_create_object(

                self.database, EndPointModel, str_endpoint_key, endpoint_attributes)

            #db_endpoint, updated = result

            db_endpoint, endpoint_updated = result



            set_kpi_sample_types(self.database, db_endpoint, endpoint.kpi_sample_types)



        event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

        dict_device_id = db_device.dump_id()
@@ -296,7 +298,10 @@ class ContextServiceServicerImpl(ContextServiceServicer): 
        dict_device_id = db_device.dump_id()



        for db_endpoint_pk,_ in db_device.references(EndPointModel):

            EndPointModel(self.database, db_endpoint_pk).delete()

            db_endpoint = EndPointModel(self.database, db_endpoint_pk)

            for db_kpi_sample_type_pk,_ in db_endpoint.references(KpiSampleTypeModel):

                KpiSampleTypeModel(self.database, db_kpi_sample_type_pk).delete()

            db_endpoint.delete()



        for db_topology_device_pk,_ in db_device.references(TopologyDeviceModel):

            TopologyDeviceModel(self.database, db_topology_device_pk).delete()

src/context/tests/Tools.py

0 → 100644
+27 −0
Original line number Diff line number Diff line 
import json

from copy import deepcopy

from typing import Any, Dict, Union

from context.proto.context_pb2 import ConfigActionEnum



def config_rule(action : ConfigActionEnum, resource_key : str, resource_value : Union[str, Dict[str, Any]]):

    if not isinstance(resource_value, str): resource_value = json.dumps(resource_value, sort_keys=True)

    return {'action': action, 'resource_key': resource_key, 'resource_value': resource_value}



def config_rule_set(resource_key : str, resource_value : Union[str, Dict[str, Any]]):

    return config_rule(ConfigActionEnum.CONFIGACTION_SET, resource_key, resource_value)



def config_rule_delete(resource_key : str, resource_value : Union[str, Dict[str, Any]]):

    return config_rule(ConfigActionEnum.CONFIGACTION_DELETE, resource_key, resource_value)



def endpoint_id(device_id, endpoint_uuid, topology_id=None):

    result = {'device_id': deepcopy(device_id), 'endpoint_uuid': {'uuid': endpoint_uuid}}

    if topology_id is not None: result['topology_id'] = deepcopy(topology_id)

    return result



def endpoint(device_id, endpoint_uuid, endpoint_type, topology_id=None, kpi_sample_types=[]):

    result = {

        'endpoint_id': endpoint_id(device_id, endpoint_uuid, topology_id=topology_id),

        'endpoint_type': endpoint_type,

    }

    if len(kpi_sample_types) > 0: result['kpi_sample_types'] = deepcopy(kpi_sample_types)

    return result

src/context/tests/example_objects.py

+56 −51
Original line number Diff line number Diff line 
from copy import deepcopy

from common.Constants import DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID

from context.proto.context_pb2 import (

    ConfigActionEnum, DeviceDriverEnum, DeviceOperationalStatusEnum, ServiceStatusEnum, ServiceTypeEnum)

    DeviceDriverEnum, DeviceOperationalStatusEnum, ServiceStatusEnum, ServiceTypeEnum)

from context.proto.kpi_sample_types_pb2 import KpiSampleType

from .Tools import config_rule_set, endpoint, endpoint_id



# Some example objects to be used by the tests



# Helper methods

def config_rule(action, resource_key, resource_value):

    return {'action': action, 'resource_key': resource_key, 'resource_value': resource_value}



def endpoint_id(topology_id, device_id, endpoint_uuid):

    return {'topology_id': deepcopy(topology_id), 'device_id': deepcopy(device_id),

            'endpoint_uuid': {'uuid': endpoint_uuid}}



def endpoint(topology_id, device_id, endpoint_uuid, endpoint_type):

    return {'endpoint_id': endpoint_id(topology_id, device_id, endpoint_uuid), 'endpoint_type': endpoint_type}



## use "deepcopy" to prevent propagating forced changes during tests

CONTEXT_ID = {'context_uuid': {'uuid': DEFAULT_CONTEXT_UUID}}

CONTEXT = {
@@ -34,22 +25,36 @@ TOPOLOGY = { 
    'link_ids': [],

}



PACKET_PORT_SAMPLE_TYPES = [

    KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED,

    KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED,

    KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED,

    KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED,

]



def _endpoint_id(device_id, endpoint_uuid):

    return endpoint_id(device_id, endpoint_uuid, topology_id=TOPOLOGY_ID)



def _endpoint(device_id, endpoint_uuid, endpoint_type):

    return endpoint(

        device_id, endpoint_uuid, endpoint_type, topology_id=TOPOLOGY_ID, kpi_sample_types=PACKET_PORT_SAMPLE_TYPES)



DEVICE1_UUID = 'DEV1'

DEVICE1_ID = {'device_uuid': {'uuid': DEVICE1_UUID}}

DEVICE1 = {

    'device_id': deepcopy(DEVICE1_ID),

    'device_type': 'packet-router',

    'device_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc1/value', 'value1'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc2/value', 'value2'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc3/value', 'value3'),

        config_rule_set('dev/rsrc1/value', 'value1'),

        config_rule_set('dev/rsrc2/value', 'value2'),

        config_rule_set('dev/rsrc3/value', 'value3'),

    ]},

    'device_operational_status': DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED,

    'device_drivers': [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_P4],

    'device_endpoints': [

        endpoint(TOPOLOGY_ID, DEVICE1_ID, 'EP2', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE1_ID, 'EP3', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE1_ID, 'EP100', 'port-packet-10G'),

        _endpoint(DEVICE1_ID, 'EP2',   'port-packet-100G'),

        _endpoint(DEVICE1_ID, 'EP3',   'port-packet-100G'),

        _endpoint(DEVICE1_ID, 'EP100', 'port-packet-10G' ),

    ],

}
@@ -59,16 +64,16 @@ DEVICE2 = { 
    'device_id': deepcopy(DEVICE2_ID),

    'device_type': 'packet-router',

    'device_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc1/value', 'value4'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc2/value', 'value5'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc3/value', 'value6'),

        config_rule_set('dev/rsrc1/value', 'value4'),

        config_rule_set('dev/rsrc2/value', 'value5'),

        config_rule_set('dev/rsrc3/value', 'value6'),

    ]},

    'device_operational_status': DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED,

    'device_drivers': [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_P4],

    'device_endpoints': [

        endpoint(TOPOLOGY_ID, DEVICE2_ID, 'EP1', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE2_ID, 'EP3', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE2_ID, 'EP100', 'port-packet-10G'),

        _endpoint(DEVICE2_ID, 'EP1',   'port-packet-100G'),

        _endpoint(DEVICE2_ID, 'EP3',   'port-packet-100G'),

        _endpoint(DEVICE2_ID, 'EP100', 'port-packet-10G' ),

    ],

}
@@ -78,16 +83,16 @@ DEVICE3 = { 
    'device_id': deepcopy(DEVICE3_ID),

    'device_type': 'packet-router',

    'device_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc1/value', 'value4'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc2/value', 'value5'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'dev/rsrc3/value', 'value6'),

        config_rule_set('dev/rsrc1/value', 'value4'),

        config_rule_set('dev/rsrc2/value', 'value5'),

        config_rule_set('dev/rsrc3/value', 'value6'),

    ]},

    'device_operational_status': DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED,

    'device_drivers': [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_P4],

    'device_endpoints': [

        endpoint(TOPOLOGY_ID, DEVICE3_ID, 'EP1', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE3_ID, 'EP2', 'port-packet-100G'),

        endpoint(TOPOLOGY_ID, DEVICE3_ID, 'EP100', 'port-packet-10G'),

        _endpoint(DEVICE3_ID, 'EP1',   'port-packet-100G'),

        _endpoint(DEVICE3_ID, 'EP2',   'port-packet-100G'),

        _endpoint(DEVICE3_ID, 'EP100', 'port-packet-10G' ),

    ],

}
@@ -96,8 +101,8 @@ LINK_DEV1_DEV2_ID = {'link_uuid': {'uuid': LINK_DEV1_DEV2_UUID}} 
LINK_DEV1_DEV2 = {

    'link_id': deepcopy(LINK_DEV1_DEV2_ID),

    'link_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE1_ID, 'EP2'),

        endpoint_id(TOPOLOGY_ID, DEVICE2_ID, 'EP1'),

        _endpoint_id(DEVICE1_ID, 'EP2'),

        _endpoint_id(DEVICE2_ID, 'EP1'),

    ]

}
@@ -106,8 +111,8 @@ LINK_DEV2_DEV3_ID = {'link_uuid': {'uuid': LINK_DEV2_DEV3_UUID}} 
LINK_DEV2_DEV3 = {

    'link_id': deepcopy(LINK_DEV2_DEV3_ID),

    'link_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE2_ID, 'EP3'),

        endpoint_id(TOPOLOGY_ID, DEVICE3_ID, 'EP2'),

        _endpoint_id(DEVICE2_ID, 'EP3'),

        _endpoint_id(DEVICE3_ID, 'EP2'),

    ]

}
@@ -116,8 +121,8 @@ LINK_DEV1_DEV3_ID = {'link_uuid': {'uuid': LINK_DEV1_DEV3_UUID}} 
LINK_DEV1_DEV3 = {

    'link_id': deepcopy(LINK_DEV1_DEV3_ID),

    'link_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE1_ID, 'EP3'),

        endpoint_id(TOPOLOGY_ID, DEVICE3_ID, 'EP1'),

        _endpoint_id(DEVICE1_ID, 'EP3'),

        _endpoint_id(DEVICE3_ID, 'EP1'),

    ]

}
@@ -130,8 +135,8 @@ SERVICE_DEV1_DEV2 = { 
    'service_id': deepcopy(SERVICE_DEV1_DEV2_ID),

    'service_type': ServiceTypeEnum.SERVICETYPE_L3NM,

    'service_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE1_ID, 'EP100'),

        endpoint_id(TOPOLOGY_ID, DEVICE2_ID, 'EP100'),

        _endpoint_id(DEVICE1_ID, 'EP100'),

        _endpoint_id(DEVICE2_ID, 'EP100'),

    ],

    'service_constraints': [

        {'constraint_type': 'latency_ms', 'constraint_value': '15.2'},
@@ -139,9 +144,9 @@ SERVICE_DEV1_DEV2 = { 
    ],

    'service_status': {'service_status': ServiceStatusEnum.SERVICESTATUS_ACTIVE},

    'service_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc1/value', 'value7'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc2/value', 'value8'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc3/value', 'value9'),

        config_rule_set('svc/rsrc1/value', 'value7'),

        config_rule_set('svc/rsrc2/value', 'value8'),

        config_rule_set('svc/rsrc3/value', 'value9'),

    ]},

}
@@ -154,8 +159,8 @@ SERVICE_DEV1_DEV3 = { 
    'service_id': deepcopy(SERVICE_DEV1_DEV3_ID),

    'service_type': ServiceTypeEnum.SERVICETYPE_L3NM,

    'service_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE1_ID, 'EP100'),

        endpoint_id(TOPOLOGY_ID, DEVICE3_ID, 'EP100'),

        _endpoint_id(DEVICE1_ID, 'EP100'),

        _endpoint_id(DEVICE3_ID, 'EP100'),

    ],

    'service_constraints': [

        {'constraint_type': 'latency_ms', 'constraint_value': '5.8'},
@@ -163,9 +168,9 @@ SERVICE_DEV1_DEV3 = { 
    ],

    'service_status': {'service_status': ServiceStatusEnum.SERVICESTATUS_ACTIVE},

    'service_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc1/value', 'value7'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc2/value', 'value8'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc3/value', 'value9'),

        config_rule_set('svc/rsrc1/value', 'value7'),

        config_rule_set('svc/rsrc2/value', 'value8'),

        config_rule_set('svc/rsrc3/value', 'value9'),

    ]},

}
@@ -178,8 +183,8 @@ SERVICE_DEV2_DEV3 = { 
    'service_id': deepcopy(SERVICE_DEV2_DEV3_ID),

    'service_type': ServiceTypeEnum.SERVICETYPE_L3NM,

    'service_endpoint_ids' : [

        endpoint_id(TOPOLOGY_ID, DEVICE2_ID, 'EP100'),

        endpoint_id(TOPOLOGY_ID, DEVICE3_ID, 'EP100'),

        _endpoint_id(DEVICE2_ID, 'EP100'),

        _endpoint_id(DEVICE3_ID, 'EP100'),

    ],

    'service_constraints': [

        {'constraint_type': 'latency_ms', 'constraint_value': '23.1'},
@@ -187,8 +192,8 @@ SERVICE_DEV2_DEV3 = { 
    ],

    'service_status': {'service_status': ServiceStatusEnum.SERVICESTATUS_ACTIVE},

    'service_config': {'config_rules': [

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc1/value', 'value7'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc2/value', 'value8'),

        config_rule(ConfigActionEnum.CONFIGACTION_SET, 'svc/rsrc3/value', 'value9'),

        config_rule_set('svc/rsrc1/value', 'value7'),

        config_rule_set('svc/rsrc2/value', 'value8'),

        config_rule_set('svc/rsrc3/value', 'value9'),

    ]},

}