Context component:

- Added basic support for ACL rules
- Corrected unitary tests

scripts/run_tests_locally-context.sh

@@ -13,19 +13,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+########################################################################################################################
+# Define your deployment settings here
+########################################################################################################################
+
+# If not already set, set the name of the Kubernetes namespace to deploy to.
+export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}
+
+export TFS_K8S_HOSTNAME="tfs-vm"
+
+########################################################################################################################
+# Automated steps start here
+########################################################################################################################
 
 PROJECTDIR=`pwd`
 
 cd $PROJECTDIR/src
 RCFILE=$PROJECTDIR/coverage/.coveragerc
 
-K8S_NAMESPACE="tf-dev"
-K8S_HOSTNAME="kubernetes-master"
-
-kubectl --namespace $K8S_NAMESPACE expose deployment contextservice --port=6379 --name=redis-tests --type=NodePort
-export REDIS_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
-export REDIS_SERVICE_PORT=$(kubectl get service redis-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')
+kubectl --namespace $TFS_K8S_NAMESPACE expose deployment contextservice --name=redis-tests --port=6379 --type=NodePort
+#export REDIS_SERVICE_HOST=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.clusterIP}')
+export REDIS_SERVICE_HOST=$(kubectl get node $TFS_K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
+export REDIS_SERVICE_PORT=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')
 
 # Run unitary tests and analyze coverage of code at same time
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose --maxfail=1 \
     context/tests/test_unitary.py
+
+kubectl --namespace $TFS_K8S_NAMESPACE delete service redis-tests

src/context/service/database/ConfigModel.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import functools, logging, operator
+import functools, json, logging, operator
 from enum import Enum
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Type, Union
 from common.orm.Database import Database
 from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object
 from common.orm.backend.Tools import key_to_str
@@ -24,8 +24,9 @@ from common.orm.fields.IntegerField import IntegerField
 from common.orm.fields.PrimaryKeyField import PrimaryKeyField
 from common.orm.fields.StringField import StringField
 from common.orm.model.Model import Model
-from common.proto.context_pb2 import ConfigActionEnum
+from common.proto.context_pb2 import ConfigActionEnum, ConfigRule
 from common.tools.grpc.Tools import grpc_message_to_json_string
+#from .EndPointModel import EndPointModel, get_endpoint
 from .Tools import fast_hasher, grpc_to_enum, remove_dict_key
 
 LOGGER = logging.getLogger(__name__)
@@ -52,86 +53,176 @@ class ConfigModel(Model): # pylint: disable=abstract-method
         config_rules = sorted(config_rules, key=operator.itemgetter('position'))
         return [remove_dict_key(config_rule, 'position') for config_rule in config_rules]
 
+class ConfigRuleCustomModel(Model): # pylint: disable=abstract-method
+    key = StringField(required=True, allow_empty=False)
+    value = StringField(required=True, allow_empty=False)
+
+    def dump(self) -> Dict: # pylint: disable=arguments-differ
+        return {'custom': {'resource_key': self.key, 'resource_value': self.value}}
+
+class ConfigRuleAclModel(Model): # pylint: disable=abstract-method
+    # TODO: improve definition of fields in ConfigRuleAclModel
+    # To simplify, endpoint encoded as JSON-string directly; otherwise causes circular dependencies
+    #endpoint_fk = ForeignKeyField(EndPointModel)
+    endpoint_id = StringField(required=True, allow_empty=False)
+    # To simplify, ACL rule is encoded as a JSON-string directly
+    acl_data = StringField(required=True, allow_empty=False)
+
+    def dump(self) -> Dict: # pylint: disable=arguments-differ
+        #json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id()
+        json_endpoint_id = json.loads(self.endpoint_id)
+        json_acl_rule_set = json.loads(self.acl_data)
+        return {'acl': {'endpoint_id': json_endpoint_id, 'rule_set': json_acl_rule_set}}
+
+# enum values should match name of field in ConfigRuleModel
+class ConfigRuleKindEnum(Enum):
+    CUSTOM = 'custom'
+    ACL    = 'acl'
+
+Union_SpecificConfigRule = Union[
+    ConfigRuleCustomModel, ConfigRuleAclModel
+]
+
 class ConfigRuleModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     config_fk = ForeignKeyField(ConfigModel)
+    kind = EnumeratedField(ConfigRuleKindEnum)
     position = IntegerField(min_value=0, required=True)
     action = EnumeratedField(ORM_ConfigActionEnum, required=True)
-    key = StringField(required=True, allow_empty=False)
-    value = StringField(required=True, allow_empty=False)
+    config_rule_custom_fk = ForeignKeyField(ConfigRuleCustomModel, required=False)
+    config_rule_acl_fk    = ForeignKeyField(ConfigRuleAclModel, required=False)
+
+    def delete(self) -> None:
+        field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value))
+        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)
+        if specific_fk_value is None:
+            raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name))
+        specific_fk_class = getattr(ConfigRuleModel, field_name, None)
+        foreign_model_class : Model = specific_fk_class.foreign_model
+        super().delete()
+        get_object(self.database, foreign_model_class, str(specific_fk_value)).delete()
 
     def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ
-        result = {
-            'action': self.action.value,
-            'custom': {
-                'resource_key': self.key,
-                'resource_value': self.value,
-            },
-        }
+        field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value))
+        specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None)
+        if specific_fk_value is None:
+            raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name))
+        specific_fk_class = getattr(ConfigRuleModel, field_name, None)
+        foreign_model_class : Model = specific_fk_class.foreign_model
+        config_rule : Union_SpecificConfigRule = get_object(self.database, foreign_model_class, str(specific_fk_value))
+        result = config_rule.dump()
+        result['action'] = self.action.value
         if include_position: result['position'] = self.position
         return result
 
+Tuple_ConfigRuleSpecs = Tuple[Type, str, Dict, ConfigRuleKindEnum]
+
+def parse_config_rule_custom(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs:
+    config_rule_class = ConfigRuleCustomModel
+    str_config_rule_id = grpc_config_rule.custom.resource_key
+    config_rule_data = {
+        'key'  : grpc_config_rule.custom.resource_key,
+        'value': grpc_config_rule.custom.resource_value,
+    }
+    return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.CUSTOM
+
+def parse_config_rule_acl(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs:
+    config_rule_class = ConfigRuleAclModel
+    grpc_endpoint_id = grpc_config_rule.acl.endpoint_id
+    grpc_rule_set = grpc_config_rule.acl.rule_set
+    device_uuid = grpc_endpoint_id.device_id.device_uuid.uuid
+    endpoint_uuid = grpc_endpoint_id.endpoint_uuid.uuid
+    str_endpoint_key = '/'.join([device_uuid, endpoint_uuid])
+    #str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id)
+    str_config_rule_id = ':'.join([str_endpoint_key, grpc_rule_set.name])
+    config_rule_data = {
+        #'endpoint_fk': db_endpoint,
+        'endpoint_id': grpc_message_to_json_string(grpc_endpoint_id),
+        'acl_data': grpc_message_to_json_string(grpc_rule_set),
+    }
+    return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.ACL
+
+CONFIGRULE_PARSERS = {
+    'custom': parse_config_rule_custom,
+    'acl'   : parse_config_rule_acl,
+}
+
+Union_ConfigRuleModel = Union[
+    ConfigRuleCustomModel, ConfigRuleAclModel,
+]
+
 def set_config_rule(
-    database : Database, db_config : ConfigModel, position : int, resource_key : str, resource_value : str
-) -> Tuple[ConfigRuleModel, bool]:
-
-    str_rule_key_hash = fast_hasher(resource_key)
-    str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':')
-    result : Tuple[ConfigRuleModel, bool] = update_or_create_object(database, ConfigRuleModel, str_config_rule_key, {
-        'config_fk': db_config, 'position': position, 'action': ORM_ConfigActionEnum.SET,
-        'key': resource_key, 'value': resource_value})
+    database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule, position : int
+) -> Tuple[Union_ConfigRuleModel, bool]:
+    grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule'))
+    parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind)
+    if parser is None:
+        raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format(
+            grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule)))
+
+    # create specific ConfigRule
+    config_rule_class, str_config_rule_id, config_rule_data, config_rule_kind = parser(database, grpc_config_rule)
+    str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id]))
+    str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':')
+    result : Tuple[Union_ConfigRuleModel, bool] = update_or_create_object(
+        database, config_rule_class, str_config_rule_key, config_rule_data)
+    db_specific_config_rule, updated = result
+
+    # create generic ConfigRule
+    config_rule_fk_field_name = 'config_rule_{:s}_fk'.format(config_rule_kind.value)
+    config_rule_data = {
+        'config_fk': db_config, 'kind': config_rule_kind, 'position': position,
+        'action': ORM_ConfigActionEnum.SET,
+        config_rule_fk_field_name: db_specific_config_rule
+    }
+    result : Tuple[ConfigRuleModel, bool] = update_or_create_object(
+        database, ConfigRuleModel, str_config_rule_key, config_rule_data)
     db_config_rule, updated = result
+
     return db_config_rule, updated
 
 def delete_config_rule(
-    database : Database, db_config : ConfigModel, resource_key : str
+    database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule
 ) -> None:
-
-    str_rule_key_hash = fast_hasher(resource_key)
-    str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':')
+    grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule'))
+    parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind)
+    if parser is None:
+        raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format(
+            grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule)))
+
+    # delete generic config rules; self deletes specific config rule
+    _, str_config_rule_id, _, config_rule_kind = parser(database, grpc_config_rule)
+    str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id]))
+    str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':')
     db_config_rule : Optional[ConfigRuleModel] = get_object(
         database, ConfigRuleModel, str_config_rule_key, raise_if_not_found=False)
     if db_config_rule is None: return
     db_config_rule.delete()
 
-def delete_all_config_rules(
-    database : Database, db_config : ConfigModel
-) -> None:
-
-    db_config_rule_pks = db_config.references(ConfigRuleModel)
-    for pk,_ in db_config_rule_pks: ConfigRuleModel(database, pk).delete()
-
-def grpc_config_rules_to_raw(grpc_config_rules) -> List[Tuple[ORM_ConfigActionEnum, str, str]]:
-    def translate(grpc_config_rule):
-        action = grpc_to_enum__config_action(grpc_config_rule.action)
-        config_rule_type = str(grpc_config_rule.WhichOneof('config_rule'))
-        if config_rule_type != 'custom':
-            raise NotImplementedError('ConfigRule of type {:s} is not implemented: {:s}'.format(
-                config_rule_type, grpc_message_to_json_string(grpc_config_rule)))
-        return action, grpc_config_rule.custom.resource_key, grpc_config_rule.custom.resource_value
-    return [translate(grpc_config_rule) for grpc_config_rule in grpc_config_rules]
-
 def update_config(
-    database : Database, db_parent_pk : str, config_name : str,
-    raw_config_rules : List[Tuple[ORM_ConfigActionEnum, str, str]]
+    database : Database, db_parent_pk : str, config_name : str, grpc_config_rules
 ) -> List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]]:
 
     str_config_key = key_to_str([config_name, db_parent_pk], separator=':')
     result : Tuple[ConfigModel, bool] = get_or_create_object(database, ConfigModel, str_config_key)
     db_config, created = result
 
-    db_objects : List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]] = [(db_config, created)]
+    db_objects = [(db_config, created)]
+
+    for position,grpc_config_rule in enumerate(grpc_config_rules):
+        action = grpc_to_enum__config_action(grpc_config_rule.action)
 
-    for position,(action, resource_key, resource_value) in enumerate(raw_config_rules):
         if action == ORM_ConfigActionEnum.SET:
             result : Tuple[ConfigRuleModel, bool] = set_config_rule(
-                database, db_config, position, resource_key, resource_value)
+                database, db_config, grpc_config_rule, position)
             db_config_rule, updated = result
             db_objects.append((db_config_rule, updated))
         elif action == ORM_ConfigActionEnum.DELETE:
-            delete_config_rule(database, db_config, resource_key)
+            delete_config_rule(database, db_config, grpc_config_rule)
         else:
-            msg = 'Unsupported action({:s}) for resource_key({:s})/resource_value({:s})'
-            raise AttributeError(msg.format(str(ConfigActionEnum.Name(action)), str(resource_key), str(resource_value)))
+            msg = 'Unsupported Action({:s}) for ConfigRule({:s})'
+            str_action = str(ConfigActionEnum.Name(action))
+            str_config_rule = grpc_message_to_json_string(grpc_config_rule)
+            raise AttributeError(msg.format(str_action, str_config_rule))
 
     return db_objects

src/context/service/grpc_server/ContextServiceServicerImpl.py

@@ -31,7 +31,7 @@ from common.proto.context_pb2 import (
 from common.proto.context_pb2_grpc import ContextServiceServicer
 from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method
 from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException
-from context.service.database.ConfigModel import grpc_config_rules_to_raw, update_config
+from context.service.database.ConfigModel import update_config
 from context.service.database.ConnectionModel import ConnectionModel, set_path
 from context.service.database.ConstraintModel import set_constraints
 from context.service.database.ContextModel import ContextModel
@@ -277,8 +277,8 @@ class ContextServiceServicerImpl(ContextServiceServicer):
                         'request.device_endpoints[{:d}].device_id.device_uuid.uuid'.format(i), endpoint_device_uuid,
                         ['should be == {:s}({:s})'.format('request.device_id.device_uuid.uuid', device_uuid)])
 
-            config_rules = grpc_config_rules_to_raw(request.device_config.config_rules)
-            running_config_rules = update_config(self.database, device_uuid, 'device', config_rules)
+            running_config_rules = update_config(
+                self.database, device_uuid, 'device', request.device_config.config_rules)
             db_running_config = running_config_rules[0][0]
 
             result : Tuple[DeviceModel, bool] = update_or_create_object(self.database, DeviceModel, device_uuid, {
@@ -487,8 +487,8 @@ class ContextServiceServicerImpl(ContextServiceServicer):
                 self.database, str_service_key, 'service', request.service_constraints)
             db_constraints = constraints_result[0][0]
 
-            config_rules = grpc_config_rules_to_raw(request.service_config.config_rules)
-            running_config_rules = update_config(self.database, str_service_key, 'service', config_rules)
+            running_config_rules = update_config(
+                self.database, str_service_key, 'service', request.service_config.config_rules)
             db_running_config = running_config_rules[0][0]
 
             result : Tuple[ServiceModel, bool] = update_or_create_object(self.database, ServiceModel, str_service_key, {
@@ -596,8 +596,8 @@ class ContextServiceServicerImpl(ContextServiceServicer):
                 self.database, str_slice_key, 'slice', request.slice_constraints)
             db_constraints = constraints_result[0][0]
 
-            config_rules = grpc_config_rules_to_raw(request.slice_config.config_rules)
-            running_config_rules = update_config(self.database, str_slice_key, 'slice', config_rules)
+            running_config_rules = update_config(
+                self.database, str_slice_key, 'slice', request.slice_config.config_rules)
             db_running_config = running_config_rules[0][0]
 
             result : Tuple[SliceModel, bool] = update_or_create_object(self.database, SliceModel, str_slice_key, {

src/context/tests/test_unitary.py

@@ -50,8 +50,8 @@ LOGGER = logging.getLogger(__name__)
 LOGGER.setLevel(logging.DEBUG)
 
 LOCAL_HOST = '127.0.0.1'
-GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT)   # avoid privileged ports
-HTTP_PORT = 10000 + get_service_port_http(ServiceNameEnum.CONTEXT)   # avoid privileged ports
+GRPC_PORT = 10000 + int(get_service_port_grpc(ServiceNameEnum.CONTEXT)) # avoid privileged ports
+HTTP_PORT = 10000 + int(get_service_port_http(ServiceNameEnum.CONTEXT)) # avoid privileged ports
 
 os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST     )] = str(LOCAL_HOST)
 os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT)
@@ -459,7 +459,7 @@ def test_grpc_device(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 40
+    assert len(db_entries) == 47
 
     # ----- Get when the object exists ---------------------------------------------------------------------------------
     response = context_client_grpc.GetDevice(DeviceId(**DEVICE_R1_ID))
@@ -512,7 +512,7 @@ def test_grpc_device(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 40
+    assert len(db_entries) == 47
 
     # ----- Remove the object ------------------------------------------------------------------------------------------
     context_client_grpc.RemoveDevice(DeviceId(**DEVICE_R1_ID))
@@ -611,7 +611,7 @@ def test_grpc_link(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 67
+    assert len(db_entries) == 80
 
     # ----- Create the object ------------------------------------------------------------------------------------------
     response = context_client_grpc.SetLink(Link(**LINK_R1_R2))
@@ -639,7 +639,7 @@ def test_grpc_link(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 75
+    assert len(db_entries) == 88
 
     # ----- Get when the object exists ---------------------------------------------------------------------------------
     response = context_client_grpc.GetLink(LinkId(**LINK_R1_R2_ID))
@@ -685,7 +685,7 @@ def test_grpc_link(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 75
+    assert len(db_entries) == 88
 
     # ----- Remove the object ------------------------------------------------------------------------------------------
     context_client_grpc.RemoveLink(LinkId(**LINK_R1_R2_ID))
@@ -794,7 +794,7 @@ def test_grpc_service(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 67
+    assert len(db_entries) == 80
 
     # ----- Create the object ------------------------------------------------------------------------------------------
     with pytest.raises(grpc.RpcError) as e:
@@ -846,7 +846,7 @@ def test_grpc_service(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 89
+    assert len(db_entries) == 108
 
     # ----- Get when the object exists ---------------------------------------------------------------------------------
     response = context_client_grpc.GetService(ServiceId(**SERVICE_R1_R2_ID))
@@ -1042,7 +1042,7 @@ def test_grpc_connection(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 150
+    assert len(db_entries) == 187
 
     # ----- Create the object ------------------------------------------------------------------------------------------
     with pytest.raises(grpc.RpcError) as e:
@@ -1082,7 +1082,7 @@ def test_grpc_connection(
     for db_entry in db_entries:
         LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover
     LOGGER.info('-----------------------------------------------------------')
-    assert len(db_entries) == 166
+    assert len(db_entries) == 203
 
     # ----- Get when the object exists ---------------------------------------------------------------------------------
     response = context_client_grpc.GetConnection(ConnectionId(**CONNECTION_R1_R3_ID))