diff --git a/scripts/run_tests_locally-context.sh b/scripts/run_tests_locally-context.sh index a9e601208aa9259219708a5e1ca770232e44faa6..7033fcb01a468731b498708096a80fac8d9a9a85 100755 --- a/scripts/run_tests_locally-context.sh +++ b/scripts/run_tests_locally-context.sh @@ -13,19 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +export TFS_K8S_HOSTNAME="tfs-vm" + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## PROJECTDIR=`pwd` cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc -K8S_NAMESPACE="tf-dev" -K8S_HOSTNAME="kubernetes-master" - -kubectl --namespace $K8S_NAMESPACE expose deployment contextservice --port=6379 --name=redis-tests --type=NodePort -export REDIS_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') -export REDIS_SERVICE_PORT=$(kubectl get service redis-tests --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}') +kubectl --namespace $TFS_K8S_NAMESPACE expose deployment contextservice --name=redis-tests --port=6379 --type=NodePort +#export REDIS_SERVICE_HOST=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.clusterIP}') +export REDIS_SERVICE_HOST=$(kubectl get node $TFS_K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export REDIS_SERVICE_PORT=$(kubectl --namespace $TFS_K8S_NAMESPACE get service redis-tests -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}') # Run unitary tests and analyze coverage of code at same time -coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose --maxfail=1 \ context/tests/test_unitary.py + +kubectl --namespace $TFS_K8S_NAMESPACE delete service redis-tests diff --git a/src/context/service/database/ConfigModel.py b/src/context/service/database/ConfigModel.py index a5f90788e4783edf1eba76cf6fe461aaa96476e6..5c6ef0079a03116d4f67519440d93185b94f2969 100644 --- a/src/context/service/database/ConfigModel.py +++ b/src/context/service/database/ConfigModel.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools, logging, operator +import functools, json, logging, operator from enum import Enum -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Type, Union from common.orm.Database import Database from common.orm.HighLevel import get_object, get_or_create_object, update_or_create_object from common.orm.backend.Tools import key_to_str @@ -24,8 +24,9 @@ from common.orm.fields.IntegerField import IntegerField from common.orm.fields.PrimaryKeyField import PrimaryKeyField from common.orm.fields.StringField import StringField from common.orm.model.Model import Model -from common.proto.context_pb2 import ConfigActionEnum +from common.proto.context_pb2 import ConfigActionEnum, ConfigRule from common.tools.grpc.Tools import grpc_message_to_json_string +#from .EndPointModel import EndPointModel, get_endpoint from .Tools import fast_hasher, grpc_to_enum, remove_dict_key LOGGER = logging.getLogger(__name__) @@ -52,86 +53,176 @@ class ConfigModel(Model): # pylint: disable=abstract-method config_rules = sorted(config_rules, key=operator.itemgetter('position')) return [remove_dict_key(config_rule, 'position') for config_rule in config_rules] +class ConfigRuleCustomModel(Model): # pylint: disable=abstract-method + key = StringField(required=True, allow_empty=False) + value = StringField(required=True, allow_empty=False) + + def dump(self) -> Dict: # pylint: disable=arguments-differ + return {'custom': {'resource_key': self.key, 'resource_value': self.value}} + +class ConfigRuleAclModel(Model): # pylint: disable=abstract-method + # TODO: improve definition of fields in ConfigRuleAclModel + # To simplify, endpoint encoded as JSON-string directly; otherwise causes circular dependencies + #endpoint_fk = ForeignKeyField(EndPointModel) + endpoint_id = StringField(required=True, allow_empty=False) + # To simplify, ACL rule is encoded as a JSON-string directly + acl_data = StringField(required=True, allow_empty=False) + + def dump(self) -> Dict: # pylint: disable=arguments-differ + #json_endpoint_id = EndPointModel(self.database, self.endpoint_fk).dump_id() + json_endpoint_id = json.loads(self.endpoint_id) + json_acl_rule_set = json.loads(self.acl_data) + return {'acl': {'endpoint_id': json_endpoint_id, 'rule_set': json_acl_rule_set}} + +# enum values should match name of field in ConfigRuleModel +class ConfigRuleKindEnum(Enum): + CUSTOM = 'custom' + ACL = 'acl' + +Union_SpecificConfigRule = Union[ + ConfigRuleCustomModel, ConfigRuleAclModel +] + class ConfigRuleModel(Model): # pylint: disable=abstract-method pk = PrimaryKeyField() config_fk = ForeignKeyField(ConfigModel) + kind = EnumeratedField(ConfigRuleKindEnum) position = IntegerField(min_value=0, required=True) action = EnumeratedField(ORM_ConfigActionEnum, required=True) - key = StringField(required=True, allow_empty=False) - value = StringField(required=True, allow_empty=False) + config_rule_custom_fk = ForeignKeyField(ConfigRuleCustomModel, required=False) + config_rule_acl_fk = ForeignKeyField(ConfigRuleAclModel, required=False) + + def delete(self) -> None: + field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value)) + specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None) + if specific_fk_value is None: + raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name)) + specific_fk_class = getattr(ConfigRuleModel, field_name, None) + foreign_model_class : Model = specific_fk_class.foreign_model + super().delete() + get_object(self.database, foreign_model_class, str(specific_fk_value)).delete() def dump(self, include_position=True) -> Dict: # pylint: disable=arguments-differ - result = { - 'action': self.action.value, - 'custom': { - 'resource_key': self.key, - 'resource_value': self.value, - }, - } + field_name = 'config_rule_{:s}_fk'.format(str(self.kind.value)) + specific_fk_value : Optional[ForeignKeyField] = getattr(self, field_name, None) + if specific_fk_value is None: + raise Exception('Unable to find config_rule key for field_name({:s})'.format(field_name)) + specific_fk_class = getattr(ConfigRuleModel, field_name, None) + foreign_model_class : Model = specific_fk_class.foreign_model + config_rule : Union_SpecificConfigRule = get_object(self.database, foreign_model_class, str(specific_fk_value)) + result = config_rule.dump() + result['action'] = self.action.value if include_position: result['position'] = self.position return result +Tuple_ConfigRuleSpecs = Tuple[Type, str, Dict, ConfigRuleKindEnum] + +def parse_config_rule_custom(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs: + config_rule_class = ConfigRuleCustomModel + str_config_rule_id = grpc_config_rule.custom.resource_key + config_rule_data = { + 'key' : grpc_config_rule.custom.resource_key, + 'value': grpc_config_rule.custom.resource_value, + } + return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.CUSTOM + +def parse_config_rule_acl(database : Database, grpc_config_rule) -> Tuple_ConfigRuleSpecs: + config_rule_class = ConfigRuleAclModel + grpc_endpoint_id = grpc_config_rule.acl.endpoint_id + grpc_rule_set = grpc_config_rule.acl.rule_set + device_uuid = grpc_endpoint_id.device_id.device_uuid.uuid + endpoint_uuid = grpc_endpoint_id.endpoint_uuid.uuid + str_endpoint_key = '/'.join([device_uuid, endpoint_uuid]) + #str_endpoint_key, db_endpoint = get_endpoint(database, grpc_endpoint_id) + str_config_rule_id = ':'.join([str_endpoint_key, grpc_rule_set.name]) + config_rule_data = { + #'endpoint_fk': db_endpoint, + 'endpoint_id': grpc_message_to_json_string(grpc_endpoint_id), + 'acl_data': grpc_message_to_json_string(grpc_rule_set), + } + return config_rule_class, str_config_rule_id, config_rule_data, ConfigRuleKindEnum.ACL + +CONFIGRULE_PARSERS = { + 'custom': parse_config_rule_custom, + 'acl' : parse_config_rule_acl, +} + +Union_ConfigRuleModel = Union[ + ConfigRuleCustomModel, ConfigRuleAclModel, +] + def set_config_rule( - database : Database, db_config : ConfigModel, position : int, resource_key : str, resource_value : str -) -> Tuple[ConfigRuleModel, bool]: - - str_rule_key_hash = fast_hasher(resource_key) - str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':') - result : Tuple[ConfigRuleModel, bool] = update_or_create_object(database, ConfigRuleModel, str_config_rule_key, { - 'config_fk': db_config, 'position': position, 'action': ORM_ConfigActionEnum.SET, - 'key': resource_key, 'value': resource_value}) + database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule, position : int +) -> Tuple[Union_ConfigRuleModel, bool]: + grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule')) + parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind) + if parser is None: + raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format( + grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule))) + + # create specific ConfigRule + config_rule_class, str_config_rule_id, config_rule_data, config_rule_kind = parser(database, grpc_config_rule) + str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id])) + str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':') + result : Tuple[Union_ConfigRuleModel, bool] = update_or_create_object( + database, config_rule_class, str_config_rule_key, config_rule_data) + db_specific_config_rule, updated = result + + # create generic ConfigRule + config_rule_fk_field_name = 'config_rule_{:s}_fk'.format(config_rule_kind.value) + config_rule_data = { + 'config_fk': db_config, 'kind': config_rule_kind, 'position': position, + 'action': ORM_ConfigActionEnum.SET, + config_rule_fk_field_name: db_specific_config_rule + } + result : Tuple[ConfigRuleModel, bool] = update_or_create_object( + database, ConfigRuleModel, str_config_rule_key, config_rule_data) db_config_rule, updated = result + return db_config_rule, updated def delete_config_rule( - database : Database, db_config : ConfigModel, resource_key : str + database : Database, db_config : ConfigModel, grpc_config_rule : ConfigRule ) -> None: - - str_rule_key_hash = fast_hasher(resource_key) - str_config_rule_key = key_to_str([db_config.pk, str_rule_key_hash], separator=':') + grpc_config_rule_kind = str(grpc_config_rule.WhichOneof('config_rule')) + parser = CONFIGRULE_PARSERS.get(grpc_config_rule_kind) + if parser is None: + raise NotImplementedError('ConfigRule of kind {:s} is not implemented: {:s}'.format( + grpc_config_rule_kind, grpc_message_to_json_string(grpc_config_rule))) + + # delete generic config rules; self deletes specific config rule + _, str_config_rule_id, _, config_rule_kind = parser(database, grpc_config_rule) + str_config_rule_key_hash = fast_hasher(':'.join([config_rule_kind.value, str_config_rule_id])) + str_config_rule_key = key_to_str([db_config.pk, str_config_rule_key_hash], separator=':') db_config_rule : Optional[ConfigRuleModel] = get_object( database, ConfigRuleModel, str_config_rule_key, raise_if_not_found=False) if db_config_rule is None: return db_config_rule.delete() -def delete_all_config_rules( - database : Database, db_config : ConfigModel -) -> None: - - db_config_rule_pks = db_config.references(ConfigRuleModel) - for pk,_ in db_config_rule_pks: ConfigRuleModel(database, pk).delete() - -def grpc_config_rules_to_raw(grpc_config_rules) -> List[Tuple[ORM_ConfigActionEnum, str, str]]: - def translate(grpc_config_rule): - action = grpc_to_enum__config_action(grpc_config_rule.action) - config_rule_type = str(grpc_config_rule.WhichOneof('config_rule')) - if config_rule_type != 'custom': - raise NotImplementedError('ConfigRule of type {:s} is not implemented: {:s}'.format( - config_rule_type, grpc_message_to_json_string(grpc_config_rule))) - return action, grpc_config_rule.custom.resource_key, grpc_config_rule.custom.resource_value - return [translate(grpc_config_rule) for grpc_config_rule in grpc_config_rules] - def update_config( - database : Database, db_parent_pk : str, config_name : str, - raw_config_rules : List[Tuple[ORM_ConfigActionEnum, str, str]] + database : Database, db_parent_pk : str, config_name : str, grpc_config_rules ) -> List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]]: str_config_key = key_to_str([config_name, db_parent_pk], separator=':') result : Tuple[ConfigModel, bool] = get_or_create_object(database, ConfigModel, str_config_key) db_config, created = result - db_objects : List[Tuple[Union[ConfigModel, ConfigRuleModel], bool]] = [(db_config, created)] + db_objects = [(db_config, created)] + + for position,grpc_config_rule in enumerate(grpc_config_rules): + action = grpc_to_enum__config_action(grpc_config_rule.action) - for position,(action, resource_key, resource_value) in enumerate(raw_config_rules): if action == ORM_ConfigActionEnum.SET: result : Tuple[ConfigRuleModel, bool] = set_config_rule( - database, db_config, position, resource_key, resource_value) + database, db_config, grpc_config_rule, position) db_config_rule, updated = result db_objects.append((db_config_rule, updated)) elif action == ORM_ConfigActionEnum.DELETE: - delete_config_rule(database, db_config, resource_key) + delete_config_rule(database, db_config, grpc_config_rule) else: - msg = 'Unsupported action({:s}) for resource_key({:s})/resource_value({:s})' - raise AttributeError(msg.format(str(ConfigActionEnum.Name(action)), str(resource_key), str(resource_value))) + msg = 'Unsupported Action({:s}) for ConfigRule({:s})' + str_action = str(ConfigActionEnum.Name(action)) + str_config_rule = grpc_message_to_json_string(grpc_config_rule) + raise AttributeError(msg.format(str_action, str_config_rule)) return db_objects diff --git a/src/context/service/grpc_server/ContextServiceServicerImpl.py b/src/context/service/grpc_server/ContextServiceServicerImpl.py index 71c97bf9ffc65942993dbdd966925f27aafad9ec..88f7bd8af82009f1fc45bace87776d9cbc6d6543 100644 --- a/src/context/service/grpc_server/ContextServiceServicerImpl.py +++ b/src/context/service/grpc_server/ContextServiceServicerImpl.py @@ -31,7 +31,7 @@ from common.proto.context_pb2 import ( from common.proto.context_pb2_grpc import ContextServiceServicer from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException -from context.service.database.ConfigModel import grpc_config_rules_to_raw, update_config +from context.service.database.ConfigModel import update_config from context.service.database.ConnectionModel import ConnectionModel, set_path from context.service.database.ConstraintModel import set_constraints from context.service.database.ContextModel import ContextModel @@ -277,8 +277,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): 'request.device_endpoints[{:d}].device_id.device_uuid.uuid'.format(i), endpoint_device_uuid, ['should be == {:s}({:s})'.format('request.device_id.device_uuid.uuid', device_uuid)]) - config_rules = grpc_config_rules_to_raw(request.device_config.config_rules) - running_config_rules = update_config(self.database, device_uuid, 'device', config_rules) + running_config_rules = update_config( + self.database, device_uuid, 'device', request.device_config.config_rules) db_running_config = running_config_rules[0][0] result : Tuple[DeviceModel, bool] = update_or_create_object(self.database, DeviceModel, device_uuid, { @@ -487,8 +487,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): self.database, str_service_key, 'service', request.service_constraints) db_constraints = constraints_result[0][0] - config_rules = grpc_config_rules_to_raw(request.service_config.config_rules) - running_config_rules = update_config(self.database, str_service_key, 'service', config_rules) + running_config_rules = update_config( + self.database, str_service_key, 'service', request.service_config.config_rules) db_running_config = running_config_rules[0][0] result : Tuple[ServiceModel, bool] = update_or_create_object(self.database, ServiceModel, str_service_key, { @@ -596,8 +596,8 @@ class ContextServiceServicerImpl(ContextServiceServicer): self.database, str_slice_key, 'slice', request.slice_constraints) db_constraints = constraints_result[0][0] - config_rules = grpc_config_rules_to_raw(request.slice_config.config_rules) - running_config_rules = update_config(self.database, str_slice_key, 'slice', config_rules) + running_config_rules = update_config( + self.database, str_slice_key, 'slice', request.slice_config.config_rules) db_running_config = running_config_rules[0][0] result : Tuple[SliceModel, bool] = update_or_create_object(self.database, SliceModel, str_slice_key, { diff --git a/src/context/tests/test_unitary.py b/src/context/tests/test_unitary.py index b46c9468c56974be5c987dbbc284daae337d3c7b..3109ef13dea98d4a56d661871b1c38ee2296f890 100644 --- a/src/context/tests/test_unitary.py +++ b/src/context/tests/test_unitary.py @@ -50,8 +50,8 @@ LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) LOCAL_HOST = '127.0.0.1' -GRPC_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT) # avoid privileged ports -HTTP_PORT = 10000 + get_service_port_http(ServiceNameEnum.CONTEXT) # avoid privileged ports +GRPC_PORT = 10000 + int(get_service_port_grpc(ServiceNameEnum.CONTEXT)) # avoid privileged ports +HTTP_PORT = 10000 + int(get_service_port_http(ServiceNameEnum.CONTEXT)) # avoid privileged ports os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(GRPC_PORT) @@ -459,7 +459,7 @@ def test_grpc_device( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 40 + assert len(db_entries) == 47 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetDevice(DeviceId(**DEVICE_R1_ID)) @@ -512,7 +512,7 @@ def test_grpc_device( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 40 + assert len(db_entries) == 47 # ----- Remove the object ------------------------------------------------------------------------------------------ context_client_grpc.RemoveDevice(DeviceId(**DEVICE_R1_ID)) @@ -611,7 +611,7 @@ def test_grpc_link( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 67 + assert len(db_entries) == 80 # ----- Create the object ------------------------------------------------------------------------------------------ response = context_client_grpc.SetLink(Link(**LINK_R1_R2)) @@ -639,7 +639,7 @@ def test_grpc_link( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 75 + assert len(db_entries) == 88 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetLink(LinkId(**LINK_R1_R2_ID)) @@ -685,7 +685,7 @@ def test_grpc_link( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 75 + assert len(db_entries) == 88 # ----- Remove the object ------------------------------------------------------------------------------------------ context_client_grpc.RemoveLink(LinkId(**LINK_R1_R2_ID)) @@ -794,7 +794,7 @@ def test_grpc_service( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 67 + assert len(db_entries) == 80 # ----- Create the object ------------------------------------------------------------------------------------------ with pytest.raises(grpc.RpcError) as e: @@ -846,7 +846,7 @@ def test_grpc_service( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 89 + assert len(db_entries) == 108 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetService(ServiceId(**SERVICE_R1_R2_ID)) @@ -1042,7 +1042,7 @@ def test_grpc_connection( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 150 + assert len(db_entries) == 187 # ----- Create the object ------------------------------------------------------------------------------------------ with pytest.raises(grpc.RpcError) as e: @@ -1082,7 +1082,7 @@ def test_grpc_connection( for db_entry in db_entries: LOGGER.info(' [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover LOGGER.info('-----------------------------------------------------------') - assert len(db_entries) == 166 + assert len(db_entries) == 203 # ----- Get when the object exists --------------------------------------------------------------------------------- response = context_client_grpc.GetConnection(ConnectionId(**CONNECTION_R1_R3_ID))