Commit b3874a8a authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component: 

- added exposition of redis-tests service for testing with Redis backend
- moved delete logic from Context Servicer to ORM components
- added support for constraints of different kinds
parent 22453d72

scripts/run_tests_locally-context.sh

+1 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc 
K8S_NAMESPACE="tf-dev"

K8S_HOSTNAME="kubernetes-master"



kubectl --namespace $K8S_NAMESPACE expose deployment contextservice --port=6379 --name=redis-tests --type=NodePort

export REDIS_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')

export REDIS_SERVICE_PORT=$(kubectl get service redis-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')

src/context/service/database/ConfigModel.py

+5 −0
Original line number Diff line number Diff line
@@ -41,6 +41,11 @@ grpc_to_enum__config_action = functools.partial( 
class ConfigModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()



    def delete(self) -> None:

        db_config_rule_pks = self.references(ConfigRuleModel)

        for pk,_ in db_config_rule_pks: ConfigRuleModel(self.database, pk).delete()

        super().delete()



    def dump(self) -> List[Dict]:

        db_config_rule_pks = self.references(ConfigRuleModel)

        config_rules = [ConfigRuleModel(self.database, pk).dump(include_position=True) for pk,_ in db_config_rule_pks]

src/context/service/database/ConnectionModel.py

+19 −2
Original line number Diff line number Diff line
@@ -32,6 +32,11 @@ LOGGER = logging.getLogger(__name__) 
class PathModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()



    def delete(self) -> None:

        for db_path_hop_pk,_ in self.references(PathHopModel):

            PathHopModel(self.database, db_path_hop_pk).delete()

        super().delete()



    def dump(self) -> List[Dict]:

        db_path_hop_pks = self.references(PathHopModel)

        path_hops = [PathHopModel(self.database, pk).dump(include_position=True) for pk,_ in db_path_hop_pks]
@@ -56,6 +61,18 @@ class ConnectionModel(Model): 
    service_fk = ForeignKeyField(ServiceModel, required=False)

    path_fk = ForeignKeyField(PathModel, required=True)



    def delete(self) -> None:

        # pylint: disable=import-outside-toplevel

        from .RelationModels import ConnectionSubServiceModel



        # Do not remove sub-services automatically. They are supported by real services, so Service component should

        # deal with the correct removal workflow to deconfigure the devices.

        for db_connection_sub_service_pk,_ in self.references(ConnectionSubServiceModel):

            ConnectionSubServiceModel(self.database, db_connection_sub_service_pk).delete()



        super().delete()

        PathModel(self.database, self.path_fk).delete()



    def dump_id(self) -> Dict:

        return {

            'connection_uuid': {'uuid': self.connection_uuid},
@@ -108,7 +125,7 @@ def set_path( 


    str_path_key = connection_uuid if len(path_name) == 0 else key_to_str([connection_uuid, path_name], separator=':')

    result : Tuple[PathModel, bool] = get_or_create_object(database, PathModel, str_path_key)

    db_path, created = result

    db_path, created = result # pylint: disable=unused-variable



    db_path_hop_pks : Set[str] = set(map(operator.itemgetter(0), db_path.references(PathHopModel)))

    db_objects : List[Tuple[Union[PathModel, PathHopModel], bool]] = [db_path]
@@ -127,7 +144,7 @@ def set_path( 
        db_endpoint : EndPointModel = get_object(database, EndPointModel, str_endpoint_key)



        result : Tuple[PathHopModel, bool] = set_path_hop(database, db_path, position, db_endpoint)

        db_path_hop, updated = result

        db_path_hop, updated = result # pylint: disable=unused-variable

        db_objects.append(db_path_hop)

        db_path_hop_pks.discard(db_path_hop.instance_key)

src/context/service/database/ConstraintModel.py

+119 −79
Original line number Diff line number Diff line
@@ -13,11 +13,13 @@ 
# limitations under the License.



import logging, operator

from typing import Dict, List, Tuple, Type, Union

from enum import Enum

from typing import Dict, List, Optional, Tuple, Type, Union

from common.orm.Database import Database

from common.orm.HighLevel import get_or_create_object, update_or_create_object

from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object

from common.orm.backend.Tools import key_to_str

from common.orm.fields.BooleanField import BooleanField

from common.orm.fields.EnumeratedField import EnumeratedField

from common.orm.fields.FloatField import FloatField

from common.orm.fields.ForeignKeyField import ForeignKeyField

from common.orm.fields.IntegerField import IntegerField
@@ -34,101 +36,123 @@ LOGGER = logging.getLogger(__name__) 
class ConstraintsModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()



    def delete(self) -> None:

        db_constraint_pks = self.references(ConstraintModel)

        for pk,_ in db_constraint_pks: ConstraintModel(self.database, pk).delete()

        super().delete()



    def dump(self) -> List[Dict]:

        db_constraint_pks = self.references(ConstraintModel)

        constraints = [ConstraintModel(self.database, pk).dump(include_position=True) for pk,_ in db_constraint_pks]

        constraints = sorted(constraints, key=operator.itemgetter('position'))

        return [remove_dict_key(constraint, 'position') for constraint in constraints]



class ConstraintModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()

    constraints_fk = ForeignKeyField(ConstraintsModel)

    position = IntegerField(min_value=0, required=True)



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = {}

        if include_position: result['position'] = self.position

        return result



class ConstraintCustomModel(ConstraintModel): # pylint: disable=abstract-method

class ConstraintCustomModel(Model): # pylint: disable=abstract-method

    constraint_type = StringField(required=True, allow_empty=False)

    constraint_value = StringField(required=True, allow_empty=False)



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = {

            'custom': {

                'constraint_type': self.constraint_type,

                'constraint_value': self.constraint_value,

            },

        }

        result.update(super().dump(include_position=include_position))

        return result



class ConstraintEndpointLocationModel(ConstraintModel): # pylint: disable=abstract-method

    endpoint_fk = ForeignKeyField(EndPointModel)

    def dump(self) -> Dict: # pylint: disable=arguments-differ

        return {'custom': {'constraint_type': self.constraint_type, 'constraint_value': self.constraint_value}}



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        db_endpoints_pks = list(self.references(EndPointModel))

Union_ConstraintEndpoint = Union[

    'ConstraintEndpointLocationGpsPositionModel', 'ConstraintEndpointLocationRegionModel',

    'ConstraintEndpointPriorityModel'

]

def dump_endpoint_id(endpoint_constraint : Union_ConstraintEndpoint):

    db_endpoints_pks = list(endpoint_constraint.references(EndPointModel))

    num_endpoints = len(db_endpoints_pks)

    if num_endpoints != 1:

        raise Exception('Wrong number({:d}) of associated Endpoints with constraint'.format(num_endpoints))

        db_endpoint = EndPointModel(self.database, db_endpoints_pks[0])

        result = {'endpoint_id': db_endpoint.dump_id()}

        result.update(super().dump(include_position=include_position))

        return result

    db_endpoint = EndPointModel(endpoint_constraint.database, db_endpoints_pks[0])

    return db_endpoint.dump_id()



class ConstraintEndpointLocationRegionModel(ConstraintEndpointLocationModel): # pylint: disable=abstract-method

class ConstraintEndpointLocationRegionModel(Model): # pylint: disable=abstract-method

    endpoint_fk = ForeignKeyField(EndPointModel)

    region = StringField(required=True, allow_empty=False)



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = super().dump(include_position=include_position)

        result['endpoint_location'].setdefault('region', {}).setdefault('region', self.region)

        return result

    def dump(self) -> Dict: # pylint: disable=arguments-differ

        return {'endpoint_location': {'endpoint_id': dump_endpoint_id(self), 'region': self.region}}



class ConstraintEndpointLocationGpsPositionModel(ConstraintEndpointLocationModel): # pylint: disable=abstract-method

class ConstraintEndpointLocationGpsPositionModel(Model): # pylint: disable=abstract-method

    endpoint_fk = ForeignKeyField(EndPointModel)

    latitude = FloatField(required=True, min_value=-90.0, max_value=90.0)

    longitude = FloatField(required=True, min_value=-180.0, max_value=180.0)



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = super().dump(include_position=include_position)

        gps_position = result['endpoint_location'].setdefault('gps_position', {})

        gps_position['latitude' ] = self.latitude

        gps_position['longitude'] = self.longitude

        return result

    def dump(self) -> Dict: # pylint: disable=arguments-differ

        gps_position = {'latitude': self.latitude, 'longitude': self.longitude}

        return {'endpoint_location': {'endpoint_id': dump_endpoint_id(self), 'gps_position': gps_position}}



class ConstraintEndpointPriorityModel(ConstraintModel): # pylint: disable=abstract-method

class ConstraintEndpointPriorityModel(Model): # pylint: disable=abstract-method

    endpoint_fk = ForeignKeyField(EndPointModel)

    priority = FloatField(required=True)



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        db_endpoints_pks = list(self.references(EndPointModel))

        num_endpoints = len(db_endpoints_pks)

        if num_endpoints != 1:

            raise Exception('Wrong number({:d}) of associated Endpoints with constraint'.format(num_endpoints))

        db_endpoint = EndPointModel(self.database, db_endpoints_pks[0])

        result = {'endpoint_id': db_endpoint.dump_id(), 'priority': self.priority}

        result.update(super().dump(include_position=include_position))

        return result

    def dump(self) -> Dict: # pylint: disable=arguments-differ

        return {'endpoint_priority': {'endpoint_id': dump_endpoint_id(self), 'priority': self.priority}}



class ConstraintSlaAvailabilityModel(ConstraintModel): # pylint: disable=abstract-method

class ConstraintSlaAvailabilityModel(Model): # pylint: disable=abstract-method

    num_disjoint_paths = IntegerField(required=True, min_value=1)

    all_active = BooleanField(required=True)



    def dump(self) -> Dict: # pylint: disable=arguments-differ

        return {'sla_availability': {'num_disjoint_paths': self.num_disjoint_paths, 'all_active': self.all_active}}



# enum values should match name of field in ConstraintModel

class ConstraintKindEnum(Enum):

    CUSTOM                        = 'custom'

    ENDPOINT_LOCATION_REGION      = 'ep_loc_region'

    ENDPOINT_LOCATION_GPSPOSITION = 'ep_loc_gpspos'

    ENDPOINT_PRIORITY             = 'ep_priority'

    SLA_AVAILABILITY              = 'sla_avail'



Union_SpecificConstraint = Union[

    ConstraintCustomModel, ConstraintEndpointLocationRegionModel, ConstraintEndpointLocationGpsPositionModel,

    ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel,

]



class ConstraintModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()

    constraints_fk = ForeignKeyField(ConstraintsModel)

    kind = EnumeratedField(ConstraintKindEnum)

    position = IntegerField(min_value=0, required=True)

    constraint_custom_fk        = ForeignKeyField(ConstraintCustomModel, required=False)

    constraint_ep_loc_region_fk = ForeignKeyField(ConstraintEndpointLocationRegionModel, required=False)

    constraint_ep_loc_gpspos_fk = ForeignKeyField(ConstraintEndpointLocationGpsPositionModel, required=False)

    constraint_ep_priority_fk   = ForeignKeyField(ConstraintEndpointPriorityModel, required=False)

    constraint_sla_avail_fk     = ForeignKeyField(ConstraintSlaAvailabilityModel, required=False)



    def delete(self) -> None:

        field_name = 'constraint_{:s}_fk'.format(str(self.kind.value))

        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)

        if specific_fk_value is None:

            raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name))

        specific_fk_class = getattr(ConstraintModel, field_name, None)

        foreign_model_class : Model = specific_fk_class.foreign_model

        super().delete()

        get_object(self.database, foreign_model_class, str(specific_fk_value)).delete()



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = {'num_disjoint_paths': self.num_disjoint_paths, 'all_active': self.all_active}

        result.update(super().dump(include_position=include_position))

        field_name = 'constraint_{:s}_fk'.format(str(self.kind.value))

        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)

        if specific_fk_value is None:

            raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name))

        specific_fk_class = getattr(ConstraintModel, field_name, None)

        foreign_model_class : Model = specific_fk_class.foreign_model

        constraint : Union_SpecificConstraint = get_object(self.database, foreign_model_class, str(specific_fk_value))

        result = constraint.dump()

        if include_position: result['position'] = self.position

        return result



def parse_constraint_custom(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:

Tuple_ConstraintSpecs = Tuple[Type, str, Dict, ConstraintKindEnum]

def parse_constraint_custom(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:

    constraint_class = ConstraintCustomModel

    str_constraint_id = grpc_constraint.custom.constraint_type

    constraint_data = {

        'constraint_type' : grpc_constraint.custom.constraint_type,

        'constraint_value': grpc_constraint.custom.constraint_value,

    }

    return constraint_class, str_constraint_id, constraint_data

    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.CUSTOM



def parse_constraint_endpoint_location(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:

def parse_constraint_endpoint_location(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:

    grpc_endpoint_id = grpc_constraint.endpoint_location.endpoint_id

    str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)
@@ -140,16 +164,17 @@ def parse_constraint_endpoint_location(database : Database, grpc_constraint) -> 
    if location_kind == 'region':

        constraint_class = ConstraintEndpointLocationRegionModel

        constraint_data.update({'region': grpc_location.region})

        return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_REGION

    elif location_kind == 'gps_position':

        constraint_class = ConstraintEndpointLocationGpsPositionModel

        gps_position = grpc_location.gps_position

        constraint_data.update({'latitude': gps_position.latitude, 'longitude': gps_position.longitude})

        return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_GPSPOSITION

    else:

        MSG = 'Location kind {:s} in Constraint of kind endpoint_location is not implemented: {:s}'

        raise NotImplementedError(MSG.format(location_kind, grpc_message_to_json_string(grpc_constraint)))

    return constraint_class, str_constraint_id, constraint_data



def parse_constraint_endpoint_priority(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:

def parse_constraint_endpoint_priority(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:

    grpc_endpoint_id = grpc_constraint.endpoint_priority.endpoint_id

    str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)
@@ -158,16 +183,16 @@ def parse_constraint_endpoint_priority(database : Database, grpc_constraint) -> 
    priority = grpc_constraint.endpoint_priority.priority

    constraint_data = {'endpoint_fk': db_endpoint, 'priority': priority}



    return constraint_class, str_constraint_id, constraint_data

    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_PRIORITY



def parse_constraint_sla_availability(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:

def parse_constraint_sla_availability(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:

    constraint_class = ConstraintSlaAvailabilityModel

    str_constraint_id = ''

    constraint_data = {

        'num_disjoint_paths' : grpc_constraint.sla_availability.num_disjoint_paths,

        'all_active': grpc_constraint.sla_availability.all_active,

    }

    return constraint_class, str_constraint_id, constraint_data

    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.SLA_AVAILABILITY



CONSTRAINT_PARSERS = {

    'custom'            : parse_constraint_custom,
@@ -176,25 +201,39 @@ CONSTRAINT_PARSERS = { 
    'sla_availability'  : parse_constraint_sla_availability,

}



Union_ConstraintModel = Union[

    ConstraintCustomModel, ConstraintEndpointLocationGpsPositionModel, ConstraintEndpointLocationRegionModel,

    ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel

]



def set_constraint(

    database : Database, db_constraints : ConstraintsModel, grpc_constraint, position : int

) -> Tuple[Constraint, bool]:

    constraint_kind = str(grpc_constraint.WhichOneof('constraint'))

    database : Database, db_constraints : ConstraintsModel, grpc_constraint : Constraint, position : int

) -> Tuple[Union_ConstraintModel, bool]:

    grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint'))



    parser = CONSTRAINT_PARSERS.get(constraint_kind)

    parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind)

    if parser is None:

        raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format(

            constraint_kind, grpc_message_to_json_string(grpc_constraint)))



    constraint_class, str_constraint_id, constraint_data = parser(database, grpc_constraint)

            grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint)))



    str_constraint_key_hash = fast_hasher(':'.join([constraint_kind, str_constraint_id]))

    # create specific constraint

    constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(database, grpc_constraint)

    str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id]))

    str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':')

    constraint_data.update({'constraints_fk': db_constraints, 'position': position})

    result : Tuple[Union_ConstraintModel, bool] = update_or_create_object(

        database, constraint_class, str_constraint_key, constraint_data)

    db_specific_constraint, updated = result



    # create generic constraint

    constraint_fk_field_name = 'constraint_{:s}_fk'.format(constraint_kind.value)

    constraint_data = {

        'constraints_fk': db_constraints, 'position': position, 'kind': constraint_kind,

        constraint_fk_field_name: db_specific_constraint

    }

    result : Tuple[ConstraintModel, bool] = update_or_create_object(

        database, constraint_class, str_constraint_key, constraint_data)

        database, ConstraintModel, str_constraint_key, constraint_data)

    db_constraint, updated = result



    return db_constraint, updated



def set_constraints(
@@ -208,7 +247,8 @@ def set_constraints( 
    db_objects = [(db_constraints, created)]



    for position,grpc_constraint in enumerate(grpc_constraints):

        result : Tuple[ConstraintModel, bool] = set_constraint(database, db_constraints, grpc_constraint, position)

        result : Tuple[ConstraintModel, bool] = set_constraint(

            database, db_constraints, grpc_constraint, position)

        db_constraint, updated = result

        db_objects.append((db_constraint, updated))

src/context/service/database/DeviceModel.py

+18 −0
Original line number Diff line number Diff line
@@ -54,6 +54,24 @@ class DeviceModel(Model): 
    device_config_fk = ForeignKeyField(ConfigModel)

    device_operational_status = EnumeratedField(ORM_DeviceOperationalStatusEnum, required=True)



    def delete(self) -> None:

        # pylint: disable=import-outside-toplevel

        from .EndPointModel import EndPointModel

        from .RelationModels import TopologyDeviceModel



        for db_endpoint_pk,_ in self.references(EndPointModel):

            EndPointModel(self.database, db_endpoint_pk).delete()



        for db_topology_device_pk,_ in self.references(TopologyDeviceModel):

            TopologyDeviceModel(self.database, db_topology_device_pk).delete()



        for db_driver_pk,_ in self.references(DriverModel):

            DriverModel(self.database, db_driver_pk).delete()



        super().delete()



        ConfigModel(self.database, self.device_config_fk).delete()



    def dump_id(self) -> Dict:

        return {'device_uuid': {'uuid': self.device_uuid}}