Commit b3874a8a authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component:

- added exposition of redis-tests service for testing with Redis backend
- moved delete logic from Context Servicer to ORM components
- added support for constraints of different kinds
parent 22453d72
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc
K8S_NAMESPACE="tf-dev"
K8S_HOSTNAME="kubernetes-master"

kubectl --namespace $K8S_NAMESPACE expose deployment contextservice --port=6379 --name=redis-tests --type=NodePort
export REDIS_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
export REDIS_SERVICE_PORT=$(kubectl get service redis-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')

+5 −0
Original line number Diff line number Diff line
@@ -41,6 +41,11 @@ grpc_to_enum__config_action = functools.partial(
class ConfigModel(Model): # pylint: disable=abstract-method
    pk = PrimaryKeyField()

    def delete(self) -> None:
        db_config_rule_pks = self.references(ConfigRuleModel)
        for pk,_ in db_config_rule_pks: ConfigRuleModel(self.database, pk).delete()
        super().delete()

    def dump(self) -> List[Dict]:
        db_config_rule_pks = self.references(ConfigRuleModel)
        config_rules = [ConfigRuleModel(self.database, pk).dump(include_position=True) for pk,_ in db_config_rule_pks]
+19 −2
Original line number Diff line number Diff line
@@ -32,6 +32,11 @@ LOGGER = logging.getLogger(__name__)
class PathModel(Model): # pylint: disable=abstract-method
    pk = PrimaryKeyField()

    def delete(self) -> None:
        for db_path_hop_pk,_ in self.references(PathHopModel):
            PathHopModel(self.database, db_path_hop_pk).delete()
        super().delete()

    def dump(self) -> List[Dict]:
        db_path_hop_pks = self.references(PathHopModel)
        path_hops = [PathHopModel(self.database, pk).dump(include_position=True) for pk,_ in db_path_hop_pks]
@@ -56,6 +61,18 @@ class ConnectionModel(Model):
    service_fk = ForeignKeyField(ServiceModel, required=False)
    path_fk = ForeignKeyField(PathModel, required=True)

    def delete(self) -> None:
        # pylint: disable=import-outside-toplevel
        from .RelationModels import ConnectionSubServiceModel

        # Do not remove sub-services automatically. They are supported by real services, so Service component should
        # deal with the correct removal workflow to deconfigure the devices.
        for db_connection_sub_service_pk,_ in self.references(ConnectionSubServiceModel):
            ConnectionSubServiceModel(self.database, db_connection_sub_service_pk).delete()

        super().delete()
        PathModel(self.database, self.path_fk).delete()

    def dump_id(self) -> Dict:
        return {
            'connection_uuid': {'uuid': self.connection_uuid},
@@ -108,7 +125,7 @@ def set_path(

    str_path_key = connection_uuid if len(path_name) == 0 else key_to_str([connection_uuid, path_name], separator=':')
    result : Tuple[PathModel, bool] = get_or_create_object(database, PathModel, str_path_key)
    db_path, created = result
    db_path, created = result # pylint: disable=unused-variable

    db_path_hop_pks : Set[str] = set(map(operator.itemgetter(0), db_path.references(PathHopModel)))
    db_objects : List[Tuple[Union[PathModel, PathHopModel], bool]] = [db_path]
@@ -127,7 +144,7 @@ def set_path(
        db_endpoint : EndPointModel = get_object(database, EndPointModel, str_endpoint_key)

        result : Tuple[PathHopModel, bool] = set_path_hop(database, db_path, position, db_endpoint)
        db_path_hop, updated = result
        db_path_hop, updated = result # pylint: disable=unused-variable
        db_objects.append(db_path_hop)
        db_path_hop_pks.discard(db_path_hop.instance_key)

+119 −79
Original line number Diff line number Diff line
@@ -13,11 +13,13 @@
# limitations under the License.

import logging, operator
from typing import Dict, List, Tuple, Type, Union
from enum import Enum
from typing import Dict, List, Optional, Tuple, Type, Union
from common.orm.Database import Database
from common.orm.HighLevel import get_or_create_object, update_or_create_object
from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object
from common.orm.backend.Tools import key_to_str
from common.orm.fields.BooleanField import BooleanField
from common.orm.fields.EnumeratedField import EnumeratedField
from common.orm.fields.FloatField import FloatField
from common.orm.fields.ForeignKeyField import ForeignKeyField
from common.orm.fields.IntegerField import IntegerField
@@ -34,101 +36,123 @@ LOGGER = logging.getLogger(__name__)
class ConstraintsModel(Model): # pylint: disable=abstract-method
    pk = PrimaryKeyField()

    def delete(self) -> None:
        db_constraint_pks = self.references(ConstraintModel)
        for pk,_ in db_constraint_pks: ConstraintModel(self.database, pk).delete()
        super().delete()

    def dump(self) -> List[Dict]:
        db_constraint_pks = self.references(ConstraintModel)
        constraints = [ConstraintModel(self.database, pk).dump(include_position=True) for pk,_ in db_constraint_pks]
        constraints = sorted(constraints, key=operator.itemgetter('position'))
        return [remove_dict_key(constraint, 'position') for constraint in constraints]

class ConstraintModel(Model): # pylint: disable=abstract-method
    pk = PrimaryKeyField()
    constraints_fk = ForeignKeyField(ConstraintsModel)
    position = IntegerField(min_value=0, required=True)

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        result = {}
        if include_position: result['position'] = self.position
        return result

class ConstraintCustomModel(ConstraintModel): # pylint: disable=abstract-method
class ConstraintCustomModel(Model): # pylint: disable=abstract-method
    constraint_type = StringField(required=True, allow_empty=False)
    constraint_value = StringField(required=True, allow_empty=False)

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        result = {
            'custom': {
                'constraint_type': self.constraint_type,
                'constraint_value': self.constraint_value,
            },
        }
        result.update(super().dump(include_position=include_position))
        return result

class ConstraintEndpointLocationModel(ConstraintModel): # pylint: disable=abstract-method
    endpoint_fk = ForeignKeyField(EndPointModel)
    def dump(self) -> Dict: # pylint: disable=arguments-differ
        return {'custom': {'constraint_type': self.constraint_type, 'constraint_value': self.constraint_value}}

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        db_endpoints_pks = list(self.references(EndPointModel))
Union_ConstraintEndpoint = Union[
    'ConstraintEndpointLocationGpsPositionModel', 'ConstraintEndpointLocationRegionModel',
    'ConstraintEndpointPriorityModel'
]
def dump_endpoint_id(endpoint_constraint : Union_ConstraintEndpoint):
    db_endpoints_pks = list(endpoint_constraint.references(EndPointModel))
    num_endpoints = len(db_endpoints_pks)
    if num_endpoints != 1:
        raise Exception('Wrong number({:d}) of associated Endpoints with constraint'.format(num_endpoints))
        db_endpoint = EndPointModel(self.database, db_endpoints_pks[0])
        result = {'endpoint_id': db_endpoint.dump_id()}
        result.update(super().dump(include_position=include_position))
        return result
    db_endpoint = EndPointModel(endpoint_constraint.database, db_endpoints_pks[0])
    return db_endpoint.dump_id()

class ConstraintEndpointLocationRegionModel(ConstraintEndpointLocationModel): # pylint: disable=abstract-method
class ConstraintEndpointLocationRegionModel(Model): # pylint: disable=abstract-method
    endpoint_fk = ForeignKeyField(EndPointModel)
    region = StringField(required=True, allow_empty=False)

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        result = super().dump(include_position=include_position)
        result['endpoint_location'].setdefault('region', {}).setdefault('region', self.region)
        return result
    def dump(self) -> Dict: # pylint: disable=arguments-differ
        return {'endpoint_location': {'endpoint_id': dump_endpoint_id(self), 'region': self.region}}

class ConstraintEndpointLocationGpsPositionModel(ConstraintEndpointLocationModel): # pylint: disable=abstract-method
class ConstraintEndpointLocationGpsPositionModel(Model): # pylint: disable=abstract-method
    endpoint_fk = ForeignKeyField(EndPointModel)
    latitude = FloatField(required=True, min_value=-90.0, max_value=90.0)
    longitude = FloatField(required=True, min_value=-180.0, max_value=180.0)

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        result = super().dump(include_position=include_position)
        gps_position = result['endpoint_location'].setdefault('gps_position', {})
        gps_position['latitude' ] = self.latitude
        gps_position['longitude'] = self.longitude
        return result
    def dump(self) -> Dict: # pylint: disable=arguments-differ
        gps_position = {'latitude': self.latitude, 'longitude': self.longitude}
        return {'endpoint_location': {'endpoint_id': dump_endpoint_id(self), 'gps_position': gps_position}}

class ConstraintEndpointPriorityModel(ConstraintModel): # pylint: disable=abstract-method
class ConstraintEndpointPriorityModel(Model): # pylint: disable=abstract-method
    endpoint_fk = ForeignKeyField(EndPointModel)
    priority = FloatField(required=True)

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        db_endpoints_pks = list(self.references(EndPointModel))
        num_endpoints = len(db_endpoints_pks)
        if num_endpoints != 1:
            raise Exception('Wrong number({:d}) of associated Endpoints with constraint'.format(num_endpoints))
        db_endpoint = EndPointModel(self.database, db_endpoints_pks[0])
        result = {'endpoint_id': db_endpoint.dump_id(), 'priority': self.priority}
        result.update(super().dump(include_position=include_position))
        return result
    def dump(self) -> Dict: # pylint: disable=arguments-differ
        return {'endpoint_priority': {'endpoint_id': dump_endpoint_id(self), 'priority': self.priority}}

class ConstraintSlaAvailabilityModel(ConstraintModel): # pylint: disable=abstract-method
class ConstraintSlaAvailabilityModel(Model): # pylint: disable=abstract-method
    num_disjoint_paths = IntegerField(required=True, min_value=1)
    all_active = BooleanField(required=True)

    def dump(self) -> Dict: # pylint: disable=arguments-differ
        return {'sla_availability': {'num_disjoint_paths': self.num_disjoint_paths, 'all_active': self.all_active}}

# enum values should match name of field in ConstraintModel
class ConstraintKindEnum(Enum):
    CUSTOM                        = 'custom'
    ENDPOINT_LOCATION_REGION      = 'ep_loc_region'
    ENDPOINT_LOCATION_GPSPOSITION = 'ep_loc_gpspos'
    ENDPOINT_PRIORITY             = 'ep_priority'
    SLA_AVAILABILITY              = 'sla_avail'

Union_SpecificConstraint = Union[
    ConstraintCustomModel, ConstraintEndpointLocationRegionModel, ConstraintEndpointLocationGpsPositionModel,
    ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel,
]

class ConstraintModel(Model): # pylint: disable=abstract-method
    pk = PrimaryKeyField()
    constraints_fk = ForeignKeyField(ConstraintsModel)
    kind = EnumeratedField(ConstraintKindEnum)
    position = IntegerField(min_value=0, required=True)
    constraint_custom_fk        = ForeignKeyField(ConstraintCustomModel, required=False)
    constraint_ep_loc_region_fk = ForeignKeyField(ConstraintEndpointLocationRegionModel, required=False)
    constraint_ep_loc_gpspos_fk = ForeignKeyField(ConstraintEndpointLocationGpsPositionModel, required=False)
    constraint_ep_priority_fk   = ForeignKeyField(ConstraintEndpointPriorityModel, required=False)
    constraint_sla_avail_fk     = ForeignKeyField(ConstraintSlaAvailabilityModel, required=False)

    def delete(self) -> None:
        field_name = 'constraint_{:s}_fk'.format(str(self.kind.value))
        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)
        if specific_fk_value is None:
            raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name))
        specific_fk_class = getattr(ConstraintModel, field_name, None)
        foreign_model_class : Model = specific_fk_class.foreign_model
        super().delete()
        get_object(self.database, foreign_model_class, str(specific_fk_value)).delete()

    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
        result = {'num_disjoint_paths': self.num_disjoint_paths, 'all_active': self.all_active}
        result.update(super().dump(include_position=include_position))
        field_name = 'constraint_{:s}_fk'.format(str(self.kind.value))
        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)
        if specific_fk_value is None:
            raise Exception('Unable to find constraint key for field_name({:s})'.format(field_name))
        specific_fk_class = getattr(ConstraintModel, field_name, None)
        foreign_model_class : Model = specific_fk_class.foreign_model
        constraint : Union_SpecificConstraint = get_object(self.database, foreign_model_class, str(specific_fk_value))
        result = constraint.dump()
        if include_position: result['position'] = self.position
        return result

def parse_constraint_custom(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:
Tuple_ConstraintSpecs = Tuple[Type, str, Dict, ConstraintKindEnum]
def parse_constraint_custom(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:
    constraint_class = ConstraintCustomModel
    str_constraint_id = grpc_constraint.custom.constraint_type
    constraint_data = {
        'constraint_type' : grpc_constraint.custom.constraint_type,
        'constraint_value': grpc_constraint.custom.constraint_value,
    }
    return constraint_class, str_constraint_id, constraint_data
    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.CUSTOM

def parse_constraint_endpoint_location(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:
def parse_constraint_endpoint_location(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:
    grpc_endpoint_id = grpc_constraint.endpoint_location.endpoint_id
    str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)

@@ -140,16 +164,17 @@ def parse_constraint_endpoint_location(database : Database, grpc_constraint) ->
    if location_kind == 'region':
        constraint_class = ConstraintEndpointLocationRegionModel
        constraint_data.update({'region': grpc_location.region})
        return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_REGION
    elif location_kind == 'gps_position':
        constraint_class = ConstraintEndpointLocationGpsPositionModel
        gps_position = grpc_location.gps_position
        constraint_data.update({'latitude': gps_position.latitude, 'longitude': gps_position.longitude})
        return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_LOCATION_GPSPOSITION
    else:
        MSG = 'Location kind {:s} in Constraint of kind endpoint_location is not implemented: {:s}'
        raise NotImplementedError(MSG.format(location_kind, grpc_message_to_json_string(grpc_constraint)))
    return constraint_class, str_constraint_id, constraint_data

def parse_constraint_endpoint_priority(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:
def parse_constraint_endpoint_priority(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:
    grpc_endpoint_id = grpc_constraint.endpoint_priority.endpoint_id
    str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)

@@ -158,16 +183,16 @@ def parse_constraint_endpoint_priority(database : Database, grpc_constraint) ->
    priority = grpc_constraint.endpoint_priority.priority
    constraint_data = {'endpoint_fk': db_endpoint, 'priority': priority}

    return constraint_class, str_constraint_id, constraint_data
    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.ENDPOINT_PRIORITY

def parse_constraint_sla_availability(database : Database, grpc_constraint) -> Tuple[Type, str, Dict]:
def parse_constraint_sla_availability(database : Database, grpc_constraint) -> Tuple_ConstraintSpecs:
    constraint_class = ConstraintSlaAvailabilityModel
    str_constraint_id = ''
    constraint_data = {
        'num_disjoint_paths' : grpc_constraint.sla_availability.num_disjoint_paths,
        'all_active': grpc_constraint.sla_availability.all_active,
    }
    return constraint_class, str_constraint_id, constraint_data
    return constraint_class, str_constraint_id, constraint_data, ConstraintKindEnum.SLA_AVAILABILITY

CONSTRAINT_PARSERS = {
    'custom'            : parse_constraint_custom,
@@ -176,25 +201,39 @@ CONSTRAINT_PARSERS = {
    'sla_availability'  : parse_constraint_sla_availability,
}

Union_ConstraintModel = Union[
    ConstraintCustomModel, ConstraintEndpointLocationGpsPositionModel, ConstraintEndpointLocationRegionModel,
    ConstraintEndpointPriorityModel, ConstraintSlaAvailabilityModel
]

def set_constraint(
    database : Database, db_constraints : ConstraintsModel, grpc_constraint, position : int
) -> Tuple[Constraint, bool]:
    constraint_kind = str(grpc_constraint.WhichOneof('constraint'))
    database : Database, db_constraints : ConstraintsModel, grpc_constraint : Constraint, position : int
) -> Tuple[Union_ConstraintModel, bool]:
    grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint'))

    parser = CONSTRAINT_PARSERS.get(constraint_kind)
    parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind)
    if parser is None:
        raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format(
            constraint_kind, grpc_message_to_json_string(grpc_constraint)))

    constraint_class, str_constraint_id, constraint_data = parser(database, grpc_constraint)
            grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint)))

    str_constraint_key_hash = fast_hasher(':'.join([constraint_kind, str_constraint_id]))
    # create specific constraint
    constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(database, grpc_constraint)
    str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id]))
    str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':')
    constraint_data.update({'constraints_fk': db_constraints, 'position': position})
    result : Tuple[Union_ConstraintModel, bool] = update_or_create_object(
        database, constraint_class, str_constraint_key, constraint_data)
    db_specific_constraint, updated = result

    # create generic constraint
    constraint_fk_field_name = 'constraint_{:s}_fk'.format(constraint_kind.value)
    constraint_data = {
        'constraints_fk': db_constraints, 'position': position, 'kind': constraint_kind,
        constraint_fk_field_name: db_specific_constraint
    }
    result : Tuple[ConstraintModel, bool] = update_or_create_object(
        database, constraint_class, str_constraint_key, constraint_data)
        database, ConstraintModel, str_constraint_key, constraint_data)
    db_constraint, updated = result

    return db_constraint, updated

def set_constraints(
@@ -208,7 +247,8 @@ def set_constraints(
    db_objects = [(db_constraints, created)]

    for position,grpc_constraint in enumerate(grpc_constraints):
        result : Tuple[ConstraintModel, bool] = set_constraint(database, db_constraints, grpc_constraint, position)
        result : Tuple[ConstraintModel, bool] = set_constraint(
            database, db_constraints, grpc_constraint, position)
        db_constraint, updated = result
        db_objects.append((db_constraint, updated))

+18 −0
Original line number Diff line number Diff line
@@ -54,6 +54,24 @@ class DeviceModel(Model):
    device_config_fk = ForeignKeyField(ConfigModel)
    device_operational_status = EnumeratedField(ORM_DeviceOperationalStatusEnum, required=True)

    def delete(self) -> None:
        # pylint: disable=import-outside-toplevel
        from .EndPointModel import EndPointModel
        from .RelationModels import TopologyDeviceModel

        for db_endpoint_pk,_ in self.references(EndPointModel):
            EndPointModel(self.database, db_endpoint_pk).delete()

        for db_topology_device_pk,_ in self.references(TopologyDeviceModel):
            TopologyDeviceModel(self.database, db_topology_device_pk).delete()

        for db_driver_pk,_ in self.references(DriverModel):
            DriverModel(self.database, db_driver_pk).delete()

        super().delete()

        ConfigModel(self.database, self.device_config_fk).delete()

    def dump_id(self) -> Dict:
        return {'device_uuid': {'uuid': self.device_uuid}}

Loading