Commit 812f09fe authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component: 

- corrected ConstraintKinds in ConstraintModel
- corrected UUID composition in ConfigRules to prevent collisions
- corrected classification of config rules to upsert/delete
- removed unneeded logs
parent 37da83e2

src/context/service/database/ConfigRule.py

+21 −29
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 


import datetime, json, logging

from sqlalchemy import delete

from sqlalchemy.dialects import postgresql

#from sqlalchemy.dialects import postgresql

from sqlalchemy.dialects.postgresql import insert

from sqlalchemy.orm import Session

from typing import Dict, List, Optional, Set
@@ -44,16 +44,16 @@ def compose_config_rules_data( 
            'updated_at': now,

        }



        parent_uuid = None

        parent_kind,parent_uuid = '',None

        if device_uuid is not None:

            dict_config_rule['device_uuid'] = device_uuid

            parent_uuid = device_uuid

            parent_kind,parent_uuid = 'device',device_uuid

        elif service_uuid is not None:

            dict_config_rule['service_uuid'] = service_uuid

            parent_uuid = service_uuid

            parent_kind,parent_uuid = 'service',service_uuid

        elif slice_uuid is not None:

            dict_config_rule['slice_uuid'] = slice_uuid

            parent_uuid = slice_uuid

            parent_kind,parent_uuid = 'slice',slice_uuid

        else:

            MSG = 'Parent for ConfigRule({:s}) cannot be identified '+\

                  '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'
@@ -62,11 +62,11 @@ def compose_config_rules_data( 


        configrule_name = None

        if kind == ConfigRuleKindEnum.CUSTOM:

            configrule_name = config_rule.custom.resource_key

            configrule_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, config_rule.custom.resource_key)

        elif kind == ConfigRuleKindEnum.ACL:

            endpoint_uuid = endpoint_get_uuid(config_rule.acl.endpoint_id, allow_random=False)

            _, _, endpoint_uuid = endpoint_get_uuid(config_rule.acl.endpoint_id, allow_random=False)

            rule_set_name = config_rule.acl.rule_set.name

            configrule_name = '{:s}/{:s}'.format(endpoint_uuid, rule_set_name)

            configrule_name = '{:s}:{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid, rule_set_name)

        else:

            MSG = 'Name for ConfigRule({:s}) cannot be inferred '+\

                  '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'
@@ -87,49 +87,39 @@ def upsert_config_rules( 
    uuids_to_upsert : Dict[str, int] = dict()

    rules_to_upsert : List[Dict] = list()

    for config_rule in config_rules:

        if config_rule['action'] == ORM_ConfigActionEnum.SET:

        configrule_uuid = config_rule['configrule_uuid']

        configrule_action = config_rule['action']

        if configrule_action == ORM_ConfigActionEnum.SET:

            position = uuids_to_upsert.get(configrule_uuid)

            if position is None:

                # if not added, add it

                rules_to_upsert.append(config_rule)

                uuids_to_upsert[config_rule['configrule_uuid']] = len(rules_to_upsert) - 1

                uuids_to_upsert[configrule_uuid] = len(rules_to_upsert) - 1

            else:

                # if already added, update occurrence

                rules_to_upsert[position] = config_rule

        elif config_rule['action'] == ORM_ConfigActionEnum.DELETE:

            uuids_to_delete.add(config_rule['configrule_uuid'])

        elif configrule_action == ORM_ConfigActionEnum.DELETE:

            uuids_to_delete.add(configrule_uuid)

        else:

            MSG = 'Action for ConfigRule({:s}) is not supported '+\

                  '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'

            str_config_rule = json.dumps(config_rule)

            raise Exception(MSG.format(str_config_rule, str(device_uuid), str(service_uuid), str(slice_uuid)))



    LOGGER.warning('device_uuid={:s}'.format(str(device_uuid)))

    LOGGER.warning('service_uuid={:s}'.format(str(service_uuid)))

    LOGGER.warning('slice_uuid={:s}'.format(str(slice_uuid)))

    LOGGER.warning('uuids_to_delete={:s}'.format(str(uuids_to_delete)))

    LOGGER.warning('rules_to_upsert={:s}'.format(str(rules_to_upsert)))



    delete_affected = False

    upsert_affected = False



    if len(uuids_to_delete) > 0:

        stmt = delete(ConfigRuleModel)

        if device_uuid  is not None: stmt = stmt.where(ConfigRuleModel.device_uuid  == device_uuid )

        if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid)

        if slice_uuid   is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid   == slice_uuid  )

        stmt = stmt.where(ConfigRuleModel.configrule_uuid.in_(uuids_to_delete))



        str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})

        LOGGER.warning('raw delete stmt={:s}'.format(str(str_stmt)))



        #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})

        #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt)))

        configrule_deletes = session.execute(stmt)



        LOGGER.warning('configrule_deletes.rowcount={:s}'.format(str(configrule_deletes.rowcount)))



        #LOGGER.warning('configrule_deletes.rowcount={:s}'.format(str(configrule_deletes.rowcount)))

        delete_affected = int(configrule_deletes.rowcount) > 0



    upsert_affected = False

    if len(rules_to_upsert) > 0:

        stmt = insert(ConfigRuleModel).values(rules_to_upsert)

        stmt = stmt.on_conflict_do_update(
@@ -142,6 +132,8 @@ def upsert_config_rules( 
            )

        )

        stmt = stmt.returning(ConfigRuleModel.created_at, ConfigRuleModel.updated_at)

        #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})

        #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt)))

        configrule_updates = session.execute(stmt).fetchall()

        upsert_affected = any([(updated_at > created_at) for created_at,updated_at in configrule_updates])

src/context/service/database/Constraint.py

+83 −84
Original line number Diff line number Diff line
@@ -14,13 +14,15 @@ 


import datetime, logging

from sqlalchemy import delete

#from sqlalchemy.dialects import postgresql

from sqlalchemy.dialects.postgresql import insert

from sqlalchemy.orm import Session

from typing import Dict, List, Optional

from common.proto.context_pb2 import Constraint

from common.tools.grpc.Tools import grpc_message_to_json_string

from .models.ConstraintModel import ConstraintKindEnum, ConstraintModel

from .uuids._Builder import get_uuid_random

from .uuids._Builder import get_uuid_from_string

from .uuids.EndPoint import endpoint_get_uuid



LOGGER = logging.getLogger(__name__)
@@ -31,16 +33,50 @@ def compose_constraints_data( 
    dict_constraints : List[Dict] = list()

    for position,constraint in enumerate(constraints):

        str_kind = constraint.WhichOneof('constraint')

        kind = ConstraintKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member

        dict_constraint = {

            'constraint_uuid': get_uuid_random(),

            'position'  : position,

            'kind'           : ConstraintKindEnum._member_map_.get(str_kind.upper()), # pylint: disable=no-member

            'kind'      : kind,

            'data'      : grpc_message_to_json_string(getattr(constraint, str_kind, {})),

            'created_at': now,

            'updated_at': now,

        }

        if service_uuid is not None: dict_constraint['service_uuid'] = service_uuid

        if slice_uuid   is not None: dict_constraint['slice_uuid'  ] = slice_uuid



        parent_kind,parent_uuid = '',None

        if service_uuid is not None:

            dict_constraint['service_uuid'] = service_uuid

            parent_kind,parent_uuid = 'service',service_uuid

        elif slice_uuid is not None:

            dict_constraint['slice_uuid'] = slice_uuid

            parent_kind,parent_uuid = 'slice',slice_uuid

        else:

            MSG = 'Parent for Constraint({:s}) cannot be identified (service_uuid={:s}, slice_uuid={:s})'

            str_constraint = grpc_message_to_json_string(constraint)

            raise Exception(MSG.format(str_constraint, str(service_uuid), str(slice_uuid)))



        constraint_name = None

        if kind == ConstraintKindEnum.CUSTOM:

            constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, constraint.custom.constraint_type)

        elif kind == ConstraintKindEnum.ENDPOINT_LOCATION:

            _, _, endpoint_uuid = endpoint_get_uuid(constraint.endpoint_location.endpoint_id, allow_random=False)

            location_kind = constraint.endpoint_location.location.WhichOneof('location')

            constraint_name = '{:s}:{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid, location_kind)

        elif kind == ConstraintKindEnum.ENDPOINT_PRIORITY:

            _, _, endpoint_uuid = endpoint_get_uuid(constraint.endpoint_priority.endpoint_id, allow_random=False)

            constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid)

        elif kind in {

            ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY,

            ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION_LEVEL

        }:

            constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value)

        else:

            MSG = 'Name for Constraint({:s}) cannot be inferred (service_uuid={:s}, slice_uuid={:s})'

            str_constraint = grpc_message_to_json_string(constraint)

            raise Exception(MSG.format(str_constraint, str(service_uuid), str(slice_uuid)))



        constraint_uuid = get_uuid_from_string(constraint_name, prefix_for_name=parent_uuid)

        dict_constraint['constraint_uuid'] = constraint_uuid



        dict_constraints.append(dict_constraint)

    return dict_constraints
@@ -48,84 +84,47 @@ def upsert_constraints( 
    session : Session, constraints : List[Dict],

    service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None

) -> bool:

    # TODO: do not delete all constraints; just add-remove as needed

    uuids_to_upsert : Dict[str, int] = dict()

    rules_to_upsert : List[Dict] = list()

    for constraint in constraints:

        constraint_uuid = constraint['constraint_uuid']

        position = uuids_to_upsert.get(constraint_uuid)

        if position is None:

            # if not added, add it

            rules_to_upsert.append(constraint)

            uuids_to_upsert[constraint_uuid] = len(rules_to_upsert) - 1

        else:

            # if already added, update occurrence

            rules_to_upsert[position] = constraint



    # Delete all constraints not in uuids_to_upsert

    delete_affected = False

    if len(uuids_to_upsert) > 0:

        stmt = delete(ConstraintModel)

        if service_uuid is not None: stmt = stmt.where(ConstraintModel.service_uuid == service_uuid)

        if slice_uuid   is not None: stmt = stmt.where(ConstraintModel.slice_uuid   == slice_uuid  )

    session.execute(stmt)

        stmt = stmt.where(ConstraintModel.constraint_uuid.not_in(set(uuids_to_upsert.keys())))

        #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})

        #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt)))

        constraint_deletes = session.execute(stmt)

        LOGGER.warning('constraint_deletes.rowcount={:s}'.format(str(constraint_deletes.rowcount)))

        delete_affected = int(constraint_deletes.rowcount) > 0



    changed = False

    upsert_affected = False

    if len(constraints) > 0:

        stmt = insert(ConstraintModel).values(constraints)

        #stmt = stmt.on_conflict_do_update(

        #    index_elements=[ConstraintModel.configrule_uuid],

        #    set_=dict(

        #        updated_at = stmt.excluded.updated_at,

        #    )

        #)

        stmt = stmt.on_conflict_do_update(

            index_elements=[ConstraintModel.constraint_uuid],

            set_=dict(

                position   = stmt.excluded.position,

                data       = stmt.excluded.data,

                updated_at = stmt.excluded.updated_at,

            )

        )

        stmt = stmt.returning(ConstraintModel.created_at, ConstraintModel.updated_at)

        #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})

        #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt)))

        constraint_updates = session.execute(stmt).fetchall()

        changed = any([(updated_at > created_at) for created_at,updated_at in constraint_updates])

        upsert_affected = any([(updated_at > created_at) for created_at,updated_at in constraint_updates])



    return changed





#    def set_constraint(self, db_constraints: ConstraintsModel, grpc_constraint: Constraint, position: int

#    ) -> Tuple[Union_ConstraintModel, bool]:

#        with self.session() as session:

#

#            grpc_constraint_kind = str(grpc_constraint.WhichOneof('constraint'))

#

#            parser = CONSTRAINT_PARSERS.get(grpc_constraint_kind)

#            if parser is None:

#                raise NotImplementedError('Constraint of kind {:s} is not implemented: {:s}'.format(

#                    grpc_constraint_kind, grpc_message_to_json_string(grpc_constraint)))

#

#            # create specific constraint

#            constraint_class, str_constraint_id, constraint_data, constraint_kind = parser(grpc_constraint)

#            str_constraint_id = str(uuid.uuid4())

#            LOGGER.info('str_constraint_id: {}'.format(str_constraint_id))

#            # str_constraint_key_hash = fast_hasher(':'.join([constraint_kind.value, str_constraint_id]))

#            # str_constraint_key = key_to_str([db_constraints.pk, str_constraint_key_hash], separator=':')

#

#            # result : Tuple[Union_ConstraintModel, bool] = update_or_create_object(

#            #     database, constraint_class, str_constraint_key, constraint_data)

#            constraint_data[constraint_class.main_pk_name()] = str_constraint_id

#            db_new_constraint = constraint_class(**constraint_data)

#            result: Tuple[Union_ConstraintModel, bool] = self.database.create_or_update(db_new_constraint)

#            db_specific_constraint, updated = result

#

#            # create generic constraint

#            # constraint_fk_field_name = 'constraint_uuid'.format(constraint_kind.value)

#            constraint_data = {

#                'constraints_uuid': db_constraints.constraints_uuid, 'position': position, 'kind': constraint_kind

#            }

#

#            db_new_constraint = ConstraintModel(**constraint_data)

#            result: Tuple[Union_ConstraintModel, bool] = self.database.create_or_update(db_new_constraint)

#            db_constraint, updated = result

#

#            return db_constraint, updated

#

#    def set_constraints(self, service_uuid: str, constraints_name : str, grpc_constraints

#    ) -> List[Tuple[Union[ConstraintsModel, ConstraintModel], bool]]:

#        with self.session() as session:

#            # str_constraints_key = key_to_str([db_parent_pk, constraints_name], separator=':')

#            # result : Tuple[ConstraintsModel, bool] = get_or_create_object(database, ConstraintsModel, str_constraints_key)

#            result = session.query(ConstraintsModel).filter_by(constraints_uuid=service_uuid).one_or_none()

#            created = None

#            if result:

#                created = True

#            session.query(ConstraintsModel).filter_by(constraints_uuid=service_uuid).one_or_none()

#            db_constraints = ConstraintsModel(constraints_uuid=service_uuid)

#            session.add(db_constraints)

#

#            db_objects = [(db_constraints, created)]

#

#            for position,grpc_constraint in enumerate(grpc_constraints):

#                result : Tuple[ConstraintModel, bool] = self.set_constraint(

#                    db_constraints, grpc_constraint, position)

#                db_constraint, updated = result

#                db_objects.append((db_constraint, updated))

#

#            return db_objects
 
    return delete_affected or upsert_affected

src/context/service/database/models/ConstraintModel.py

+8 −356

File changed.

Preview size limit exceeded, changes collapsed.