Commit 7d3e613d authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component: 

- Added basic support for ACL rules
- Corrected unitary tests
parent fb78e7e2

scripts/run_tests_locally-context.sh

+19 −7
Original line number Diff line number Diff line
@@ -13,19 +13,31 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



########################################################################################################################

# Define your deployment settings here

########################################################################################################################



# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}



export TFS_K8S_HOSTNAME="tfs-vm"



########################################################################################################################

# Automated steps start here

########################################################################################################################



PROJECTDIR=`pwd`



cd $PROJECTDIR/src

RCFILE=$PROJECTDIR/coverage/.coveragerc



K8S_NAMESPACE="tf-dev"

K8S_HOSTNAME="kubernetes-master"



kubectl --namespace $K8S_NAMESPACE expose deployment contextservice --port=6379 --name=redis-tests --type=NodePort

export REDIS_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')

export REDIS_SERVICE_PORT=$(kubectl get service redis-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')

kubectl --namespace $TFS_K8S_NAMESPACE expose deployment contextservice --name=redis-tests --port=6379 --type=NodePort

#export REDIS_SERVICE_HOST=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.clusterIP}')

export REDIS_SERVICE_HOST=$(kubectl get node $TFS_K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')

export REDIS_SERVICE_PORT=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')



# Run unitary tests and analyze coverage of code at same time

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose --maxfail=1 \

    context/tests/test_unitary.py



kubectl --namespace $TFS_K8S_NAMESPACE delete service redis-tests

src/context/service/database/ConfigModel.py

+140 −49
Original line number Diff line number Diff line
@@ -12,9 +12,9 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import functools, logging, operator

import functools, json, logging, operator

from enum import Enum

from typing import Dict, List, Optional, Tuple, Union

from typing import Dict, List, Optional, Tuple, Type, Union

from common.orm.Database import Database

from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object

from common.orm.backend.Tools import key_to_str
@@ -24,8 +24,9 @@ from common.orm.fields.IntegerField import IntegerField 
from common.orm.fields.PrimaryKeyField import PrimaryKeyField

from common.orm.fields.StringField import StringField

from common.orm.model.Model import Model

from common.proto.context_pb2 import ConfigActionEnum

from common.proto.context_pb2 import ConfigActionEnum, ConfigRule

from common.tools.grpc.Tools import grpc_message_to_json_string

#from .EndPointModel import EndPointModel, get_endpoint

from .Tools import fast_hasher, grpc_to_enum, remove_dict_key



LOGGER = logging.getLogger(__name__)
@@ -52,86 +53,176 @@ class ConfigModel(Model): # pylint: disable=abstract-method 
        config_rules = sorted(config_rules, key=operator.itemgetter('position'))

        return [remove_dict_key(config_rule, 'position') for config_rule in config_rules]



class ConfigRuleCustomModel(Model): # pylint: disable=abstract-method

    key = StringField(required=True, allow_empty=False)

    value = StringField(required=True, allow_empty=False)



    def dump(self) -> Dict: # pylint: disable=arguments-differ

        return {'custom': {'resource_key': self.key, 'resource_value': self.value}}



class ConfigRuleAclModel(Model): # pylint: disable=abstract-method

    # TODO: improve definition of fields in ConfigRuleAclModel

    # To simplify, endpoint encoded as JSON-string directly; otherwise causes circular dependencies

    #endpoint_fk = ForeignKeyField(EndPointModel)

    endpoint_id = StringField(required=True, allow_empty=False)

    # To simplify, ACL rule is encoded as a JSON-string directly

    acl_data = StringField(required=True, allow_empty=False)



    def dump(self) -> Dict: # pylint: disable=arguments-differ

        #json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id()

        json_endpoint_id = json.loads(self.endpoint_id)

        json_acl_rule_set = json.loads(self.acl_data)

        return {'acl': {'endpoint_id': json_endpoint_id, 'rule_set': json_acl_rule_set}}



# enum values should match name of field in ConfigRuleModel

class ConfigRuleKindEnum(Enum):

    CUSTOM = 'custom'

    ACL    = 'acl'



Union_SpecificConfigRule = Union[

    ConfigRuleCustomModel, ConfigRuleAclModel

]



class ConfigRuleModel(Model): # pylint: disable=abstract-method

    pk = PrimaryKeyField()

    config_fk = ForeignKeyField(ConfigModel)

    kind = EnumeratedField(ConfigRuleKindEnum)

    position = IntegerField(min_value=0, required=True)

    action = EnumeratedField(ORM_ConfigActionEnum, required=True)

    key = StringField(required=True, allow_empty=False)

    value = StringField(required=True, allow_empty=False)

    config_rule_custom_fk = ForeignKeyField(ConfigRuleCustomModel, required=False)

    config_rule_acl_fk    = ForeignKeyField(ConfigRuleAclModel, required=False)



    def delete(self) -> None:

        field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value))

        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)

        if specific_fk_value is None:

            raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name))

        specific_fk_class = getattr(ConfigRuleModel, field_name, None)

        foreign_model_class : Model = specific_fk_class.foreign_model

        super().delete()

        get_object(self.database, foreign_model_class, str(specific_fk_value)).delete()



    def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ

        result = {

            'action': self.action.value,

            'custom': {

                'resource_key': self.key,

                'resource_value': self.value,

            },

        }

        field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value))

        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)

        if specific_fk_value is None:

            raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name))

        specific_fk_class = getattr(ConfigRuleModel, field_name, None)

        foreign_model_class : Model = specific_fk_class.foreign_model

        config_rule : Union_SpecificConfigRule = get_object(self.database, foreign_model_class, str(specific_fk_value))

        result = config_rule.dump()

        result['action'] = self.action.value

        if include_position: result['position'] = self.position

        return result



Tuple_ConfigRuleSpecs = Tuple[Type, str, Dict, ConfigRuleKindEnum]



def parse_config_rule_custom(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs:

    config_rule_class = ConfigRuleCustomModel

    str_config_rule_id = grpc_config_rule.custom.resource_key

    config_rule_data = {

        'key'  : grpc_config_rule.custom.resource_key,

        'value': grpc_config_rule.custom.resource_value,

    }

    return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.CUSTOM



def parse_config_rule_acl(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs:

    config_rule_class = ConfigRuleAclModel

    grpc_endpoint_id = grpc_config_rule.acl.endpoint_id

    grpc_rule_set = grpc_config_rule.acl.rule_set

    device_uuid = grpc_endpoint_id.device_id.device_uuid.uuid

    endpoint_uuid = grpc_endpoint_id.endpoint_uuid.uuid

    str_endpoint_key = '/'.join([device_uuid, endpoint_uuid])

    #str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)

    str_config_rule_id = ':'.join([str_endpoint_key, grpc_rule_set.name])

    config_rule_data = {

        #'endpoint_fk': db_endpoint,

        'endpoint_id': grpc_message_to_json_string(grpc_endpoint_id),

        'acl_data': grpc_message_to_json_string(grpc_rule_set),

    }

    return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.ACL



CONFIGRULE_PARSERS = {

    'custom': parse_config_rule_custom,

    'acl'   : parse_config_rule_acl,

}



Union_ConfigRuleModel = Union[

    ConfigRuleCustomModel, ConfigRuleAclModel,

]



def set_config_rule(

    database : Database, db_config : ConfigModel, position : int, resource_key : str, resource_value : str

) -> Tuple[ConfigRuleModel, bool]:



    str_rule_key_hash = fast_hasher(resource_key)

    str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':')

    result : Tuple[ConfigRuleModel, bool] = update_or_create_object(database, ConfigRuleModel, str_config_rule_key, {

        'config_fk': db_config, 'position': position, 'action': ORM_ConfigActionEnum.SET,

        'key': resource_key, 'value': resource_value})

    database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule, position : int

) -> Tuple[Union_ConfigRuleModel, bool]:

    grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule'))

    parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind)

    if parser is None:

        raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format(

            grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule)))



    # create specific ConfigRule

    config_rule_class, str_config_rule_id, config_rule_data, config_rule_kind = parser(database, grpc_config_rule)

    str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id]))

    str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':')

    result : Tuple[Union_ConfigRuleModel, bool] = update_or_create_object(

        database, config_rule_class, str_config_rule_key, config_rule_data)

    db_specific_config_rule, updated = result



    # create generic ConfigRule

    config_rule_fk_field_name = 'config_rule_{:s}_fk'.format(config_rule_kind.value)

    config_rule_data = {

        'config_fk': db_config, 'kind': config_rule_kind, 'position': position,

        'action': ORM_ConfigActionEnum.SET,

        config_rule_fk_field_name: db_specific_config_rule

    }

    result : Tuple[ConfigRuleModel, bool] = update_or_create_object(

        database, ConfigRuleModel, str_config_rule_key, config_rule_data)

    db_config_rule, updated = result



    return db_config_rule, updated



def delete_config_rule(

    database : Database, db_config : ConfigModel, resource_key : str

    database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule

) -> None:



    str_rule_key_hash = fast_hasher(resource_key)

    str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':')

    grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule'))

    parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind)

    if parser is None:

        raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format(

            grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule)))



    # delete generic config rules; self deletes specific config rule

    _, str_config_rule_id, _, config_rule_kind = parser(database, grpc_config_rule)

    str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id]))

    str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':')

    db_config_rule : Optional[ConfigRuleModel] = get_object(

        database, ConfigRuleModel, str_config_rule_key, raise_if_not_found=False)

    if db_config_rule is None: return

    db_config_rule.delete()



def delete_all_config_rules(

    database : Database, db_config : ConfigModel

) -> None:



    db_config_rule_pks = db_config.references(ConfigRuleModel)

    for pk,_ in db_config_rule_pks: ConfigRuleModel(database, pk).delete()



def grpc_config_rules_to_raw(grpc_config_rules) -> List[Tuple[ORM_ConfigActionEnum, str, str]]:

    def translate(grpc_config_rule):

        action = grpc_to_enum__config_action(grpc_config_rule.action)

        config_rule_type = str(grpc_config_rule.WhichOneof('config_rule'))

        if config_rule_type != 'custom':

            raise NotImplementedError('ConfigRule of type {:s} is not implemented: {:s}'.format(

                config_rule_type, grpc_message_to_json_string(grpc_config_rule)))

        return action, grpc_config_rule.custom.resource_key, grpc_config_rule.custom.resource_value

    return [translate(grpc_config_rule) for grpc_config_rule in grpc_config_rules]



def update_config(

    database : Database, db_parent_pk : str, config_name : str,

    raw_config_rules : List[Tuple[ORM_ConfigActionEnum, str, str]]

    database : Database, db_parent_pk : str, config_name : str, grpc_config_rules

) -> List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]]:



    str_config_key = key_to_str([config_name, db_parent_pk], separator=':')

    result : Tuple[ConfigModel, bool] = get_or_create_object(database, ConfigModel, str_config_key)

    db_config, created = result



    db_objects : List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]] = [(db_config, created)]

    db_objects = [(db_config, created)]



    for position,grpc_config_rule in enumerate(grpc_config_rules):

        action = grpc_to_enum__config_action(grpc_config_rule.action)



    for position,(action, resource_key, resource_value) in enumerate(raw_config_rules):

        if action == ORM_ConfigActionEnum.SET:

            result : Tuple[ConfigRuleModel, bool] = set_config_rule(

                database, db_config, position, resource_key, resource_value)

                database, db_config, grpc_config_rule, position)

            db_config_rule, updated = result

            db_objects.append((db_config_rule, updated))

        elif action == ORM_ConfigActionEnum.DELETE:

            delete_config_rule(database, db_config, resource_key)

            delete_config_rule(database, db_config, grpc_config_rule)

        else:

            msg = 'Unsupported action({:s}) for resource_key({:s})/resource_value({:s})'

            raise AttributeError(msg.format(str(ConfigActionEnum.Name(action)), str(resource_key), str(resource_value)))

            msg = 'Unsupported Action({:s}) for ConfigRule({:s})'

            str_action = str(ConfigActionEnum.Name(action))

            str_config_rule = grpc_message_to_json_string(grpc_config_rule)

            raise AttributeError(msg.format(str_action, str_config_rule))



    return db_objects

src/context/service/grpc_server/ContextServiceServicerImpl.py

+7 −7
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ from common.proto.context_pb2 import ( 
from common.proto.context_pb2_grpc import ContextServiceServicer

from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method

from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException

from context.service.database.ConfigModel import grpc_config_rules_to_raw, update_config

from context.service.database.ConfigModel import update_config

from context.service.database.ConnectionModel import ConnectionModel, set_path

from context.service.database.ConstraintModel import set_constraints

from context.service.database.ContextModel import ContextModel
@@ -277,8 +277,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): 
                        'request.device_endpoints[{:d}].device_id.device_uuid.uuid'.format(i), endpoint_device_uuid,

                        ['should be == {:s}({:s})'.format('request.device_id.device_uuid.uuid', device_uuid)])



            config_rules = grpc_config_rules_to_raw(request.device_config.config_rules)

            running_config_rules = update_config(self.database, device_uuid, 'device', config_rules)

            running_config_rules = update_config(

                self.database, device_uuid, 'device', request.device_config.config_rules)

            db_running_config = running_config_rules[0][0]



            result : Tuple[DeviceModel, bool] = update_or_create_object(self.database, DeviceModel, device_uuid, {
@@ -487,8 +487,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): 
                self.database, str_service_key, 'service', request.service_constraints)

            db_constraints = constraints_result[0][0]



            config_rules = grpc_config_rules_to_raw(request.service_config.config_rules)

            running_config_rules = update_config(self.database, str_service_key, 'service', config_rules)

            running_config_rules = update_config(

                self.database, str_service_key, 'service', request.service_config.config_rules)

            db_running_config = running_config_rules[0][0]



            result : Tuple[ServiceModel, bool] = update_or_create_object(self.database, ServiceModel, str_service_key, {
@@ -596,8 +596,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): 
                self.database, str_slice_key, 'slice', request.slice_constraints)

            db_constraints = constraints_result[0][0]



            config_rules = grpc_config_rules_to_raw(request.slice_config.config_rules)

            running_config_rules = update_config(self.database, str_slice_key, 'slice', config_rules)

            running_config_rules = update_config(

                self.database, str_slice_key, 'slice', request.slice_config.config_rules)

            db_running_config = running_config_rules[0][0]



            result : Tuple[SliceModel, bool] = update_or_create_object(self.database, SliceModel, str_slice_key, {

src/context/tests/test_unitary.py

+11 −11
Original line number Diff line number Diff line
@@ -50,8 +50,8 @@ LOGGER = logging.getLogger(__name__) 
LOGGER.setLevel(logging.DEBUG)



LOCAL_HOST = '127.0.0.1'

GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT)   # avoid privileged ports

HTTP_PORT = 10000 + get_service_port_http(ServiceNameEnum.CONTEXT)   # avoid privileged ports

GRPC_PORT = 10000 + int(get_service_port_grpc(ServiceNameEnum.CONTEXT)) # avoid privileged ports

HTTP_PORT = 10000 + int(get_service_port_http(ServiceNameEnum.CONTEXT)) # avoid privileged ports



os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST     )] = str(LOCAL_HOST)

os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT)
@@ -459,7 +459,7 @@ def test_grpc_device( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 40

    assert len(db_entries) == 47



    # ----- Get when the object exists ---------------------------------------------------------------------------------

    response = context_client_grpc.GetDevice(DeviceId(**DEVICE_R1_ID))
@@ -512,7 +512,7 @@ def test_grpc_device( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 40

    assert len(db_entries) == 47



    # ----- Remove the object ------------------------------------------------------------------------------------------

    context_client_grpc.RemoveDevice(DeviceId(**DEVICE_R1_ID))
@@ -611,7 +611,7 @@ def test_grpc_link( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 67

    assert len(db_entries) == 80



    # ----- Create the object ------------------------------------------------------------------------------------------

    response = context_client_grpc.SetLink(Link(**LINK_R1_R2))
@@ -639,7 +639,7 @@ def test_grpc_link( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 75

    assert len(db_entries) == 88



    # ----- Get when the object exists ---------------------------------------------------------------------------------

    response = context_client_grpc.GetLink(LinkId(**LINK_R1_R2_ID))
@@ -685,7 +685,7 @@ def test_grpc_link( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 75

    assert len(db_entries) == 88



    # ----- Remove the object ------------------------------------------------------------------------------------------

    context_client_grpc.RemoveLink(LinkId(**LINK_R1_R2_ID))
@@ -794,7 +794,7 @@ def test_grpc_service( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 67

    assert len(db_entries) == 80



    # ----- Create the object ------------------------------------------------------------------------------------------

    with pytest.raises(grpc.RpcError) as e:
@@ -846,7 +846,7 @@ def test_grpc_service( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 89

    assert len(db_entries) == 108



    # ----- Get when the object exists ---------------------------------------------------------------------------------

    response = context_client_grpc.GetService(ServiceId(**SERVICE_R1_R2_ID))
@@ -1042,7 +1042,7 @@ def test_grpc_connection( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 150

    assert len(db_entries) == 187



    # ----- Create the object ------------------------------------------------------------------------------------------

    with pytest.raises(grpc.RpcError) as e:
@@ -1082,7 +1082,7 @@ def test_grpc_connection( 
    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 166

    assert len(db_entries) == 203



    # ----- Get when the object exists ---------------------------------------------------------------------------------

    response = context_client_grpc.GetConnection(ConnectionId(**CONNECTION_R1_R3_ID))