Several changes:

Context:
- ORM models completed and tested
- Deactivated unneeded code-linting warnings
- Improved coding of fast_hasher method
- Added test unit for fast_hasher method
- Improved organization and structure of test units

run_local_tests.sh

@@ -22,7 +22,9 @@ coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
 #    centralizedcybersecurity/tests/test_unitary.py
 
 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    context/tests/test_unitary.py
+    context/tests/test_unitary_fast_hasher.py \
+    context/tests/test_unitary_grpc.py \
+    #context/tests/test_unitary_rest.py
 
 #coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
 #    device/tests/test_unitary_driverapi.py \

src/context/service/database/ConfigModel.py

@@ -20,7 +20,7 @@ class ORM_ConfigActionEnum(Enum):
 grpc_to_enum__config_action = functools.partial(
     grpc_to_enum, ConfigActionEnum, ORM_ConfigActionEnum)
 
-class ConfigModel(Model):
+class ConfigModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
 
     def dump(self) -> List[Dict]:
@@ -29,7 +29,7 @@ class ConfigModel(Model):
         config_rules = sorted(config_rules, key=operator.itemgetter('position'))
         return [remove_dict_key(config_rule, 'position') for config_rule in config_rules]
 
-class ConfigRuleModel(Model):
+class ConfigRuleModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     config_fk = ForeignKeyField(ConfigModel)
     position = IntegerField(min_value=0, required=True)

src/context/service/database/ConstraintModel.py

@@ -9,7 +9,7 @@ from context.service.database.Tools import remove_dict_key
 
 LOGGER = logging.getLogger(__name__)
 
-class ConstraintsModel(Model):
+class ConstraintsModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
 
     def dump(self) -> List[Dict]:
@@ -18,7 +18,7 @@ class ConstraintsModel(Model):
         constraints = sorted(constraints, key=operator.itemgetter('position'))
         return [remove_dict_key(constraint, 'position') for constraint in constraints]
 
-class ConstraintModel(Model):
+class ConstraintModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     constraints_fk = ForeignKeyField(ConstraintsModel)
     position = IntegerField(min_value=0, required=True)

src/context/service/database/ContextModel.py

@@ -14,12 +14,12 @@ class ContextModel(Model):
         return {'context_uuid': {'uuid': self.context_uuid}}
 
     def dump_service_ids(self) -> List[Dict]:
-        from .ServiceModel import ServiceModel
+        from .ServiceModel import ServiceModel # pylint: disable=import-outside-toplevel
         db_service_pks = self.references(ServiceModel)
         return [ServiceModel(self.database, pk).dump_id() for pk in db_service_pks]
 
     def dump_topology_ids(self) -> List[Dict]:
-        from .TopologyModel import TopologyModel
+        from .TopologyModel import TopologyModel # pylint: disable=import-outside-toplevel
         db_topology_pks = self.references(TopologyModel)
         return [TopologyModel(self.database, pk).dump_id() for pk in db_topology_pks]
 

src/context/service/database/DeviceModel.py

@@ -49,7 +49,7 @@ class DeviceModel(Model):
         return [DriverModel(self.database, pk).dump() for pk,_ in db_driver_pks]
 
     def dump_endpoints(self) -> List[Dict]:
-        from .EndPointModel import EndPointModel
+        from .EndPointModel import EndPointModel # pylint: disable=import-outside-toplevel
         db_endpoints_pks = self.references(EndPointModel)
         return [EndPointModel(self.database, pk).dump() for pk,_ in db_endpoints_pks]
 
@@ -66,7 +66,7 @@ class DeviceModel(Model):
         if include_endpoints: result['device_endpoints'] = self.dump_endpoints()
         return result
 
-class DriverModel(Model):
+class DriverModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     device_fk = ForeignKeyField(DeviceModel)
     driver = EnumeratedField(ORM_DeviceDriverEnum, required=True)

src/context/service/database/LinkModel.py

@@ -14,7 +14,7 @@ class LinkModel(Model):
         return {'link_uuid': {'uuid': self.link_uuid}}
 
     def dump_endpoint_ids(self) -> List[Dict]:
-        from .RelationModels import LinkEndPointModel
+        from .RelationModels import LinkEndPointModel # pylint: disable=import-outside-toplevel
         db_endpoints = get_related_instances(self, LinkEndPointModel, 'endpoint_fk')
         return [db_endpoint.dump_id() for db_endpoint in sorted(db_endpoints, key=operator.attrgetter('pk'))]
 

src/context/service/database/RelationModels.py

@@ -10,22 +10,22 @@ from .TopologyModel import TopologyModel
 
 LOGGER = logging.getLogger(__name__)
 
-class LinkEndPointModel(Model):
+class LinkEndPointModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     link_fk = ForeignKeyField(LinkModel)
     endpoint_fk = ForeignKeyField(EndPointModel)
 
-class ServiceEndPointModel(Model):
+class ServiceEndPointModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     service_fk = ForeignKeyField(ServiceModel)
     endpoint_fk = ForeignKeyField(EndPointModel)
 
-class TopologyDeviceModel(Model):
+class TopologyDeviceModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     topology_fk = ForeignKeyField(TopologyModel)
     device_fk = ForeignKeyField(DeviceModel)
 
-class TopologyLinkModel(Model):
+class TopologyLinkModel(Model): # pylint: disable=abstract-method
     pk = PrimaryKeyField()
     topology_fk = ForeignKeyField(TopologyModel)
     link_fk = ForeignKeyField(LinkModel)

src/context/service/database/ServiceModel.py

@@ -49,7 +49,7 @@ class ServiceModel(Model):
         }
 
     def dump_endpoint_ids(self) -> List[Dict]:
-        from .RelationModels import ServiceEndPointModel
+        from .RelationModels import ServiceEndPointModel # pylint: disable=import-outside-toplevel
         db_endpoints = get_related_instances(self, ServiceEndPointModel, 'endpoint_fk')
         return [db_endpoint.dump_id() for db_endpoint in sorted(db_endpoints, key=operator.attrgetter('pk'))]
 

src/context/service/database/Tools.py

@@ -26,15 +26,33 @@ def grpc_to_enum(grpc_enum_class, orm_enum_class : Enum, grpc_enum_value):
 # For some models, it is convenient to produce a string hash for fast comparisons of existence or modification. Method
 # fast_hasher computes configurable length (between 1 and 64 byte) hashes and retrieves them in hex representation.
 
+FASTHASHER_ITEM_ACCEPTED_FORMAT = 'Union[bytes, str]'
+FASTHASHER_DATA_ACCEPTED_FORMAT = 'Union[{fmt:s}, List[{fmt:s}], Tuple[{fmt:s}]]'.format(
+    fmt=FASTHASHER_ITEM_ACCEPTED_FORMAT)
+
 def fast_hasher(data : Union[bytes, str, List[Union[bytes, str]], Tuple[Union[bytes, str]]], digest_size : int = 8):
     hasher = hashlib.blake2b(digest_size=digest_size)
     # Do not accept sets, dicts, or other unordered dats tructures since their order is arbitrary thus producing
     # different hashes depending on the order. Consider adding support for sets or dicts with previous sorting of
     # items by their key.
-    if not isinstance(data, (list, tuple)): data = [data]
-    for item in data:
-        if isinstance(item, str): item = item.encode('UTF-8')
-        if not isinstance(item, bytes):
-            raise TypeError('data type must be Union[bytes, str, List[Union[bytes, str]], Tuple[Union[bytes, str]]]')
+
+    if isinstance(data, bytes):
+        data = [data]
+    elif isinstance(data, str):
+        data = [data.encode('UTF-8')]
+    elif isinstance(data, (list, tuple)):
+        pass
+    else:
+        msg = 'data({:s}) must be {:s}, found {:s}'
+        raise TypeError(msg.format(str(data), FASTHASHER_DATA_ACCEPTED_FORMAT, str(type(data))))
+
+    for i,item in enumerate(data):
+        if isinstance(item, str):
+            item = item.encode('UTF-8')
+        elif isinstance(item, bytes):
+            pass
+        else:
+            msg = 'data[{:d}]({:s}) must be {:s}, found {:s}'
+            raise TypeError(msg.format(i, str(item), FASTHASHER_ITEM_ACCEPTED_FORMAT, str(type(item))))
         hasher.update(item)
     return hasher.hexdigest()

src/context/service/database/TopologyModel.py

@@ -21,12 +21,12 @@ class TopologyModel(Model):
         }
 
     def dump_device_ids(self) -> List[Dict]:
-        from .RelationModels import TopologyDeviceModel
+        from .RelationModels import TopologyDeviceModel # pylint: disable=import-outside-toplevel
         db_devices = get_related_instances(self, TopologyDeviceModel, 'device_fk')
         return [db_device.dump_id() for db_device in sorted(db_devices, key=operator.attrgetter('pk'))]
 
     def dump_link_ids(self) -> List[Dict]:
-        from .RelationModels import TopologyLinkModel
+        from .RelationModels import TopologyLinkModel # pylint: disable=import-outside-toplevel
         db_links = get_related_instances(self, TopologyLinkModel, 'link_fk')
         return [db_link.dump_id() for db_link in sorted(db_links, key=operator.attrgetter('pk'))]
 

src/context/tests/example_objects.py

+import copy
+from common.Constants import DEFAULT_CONTEXT_UUID, DEFAULT_TOPOLOGY_UUID
+from context.proto.context_pb2 import (
+    ConfigActionEnum, DeviceDriverEnum, DeviceOperationalStatusEnum, ServiceStatusEnum, ServiceTypeEnum)
+
+# Some example objects to be used by the tests
+
+## use "copy.deepcopy" to prevent propagating forced changes during tests
+CONTEXT_ID = {'context_uuid': {'uuid': DEFAULT_CONTEXT_UUID}}
+CONTEXT = {'context_id': copy.deepcopy(CONTEXT_ID)}
+
+TOPOLOGY_ID = {
+    'context_id': copy.deepcopy(CONTEXT_ID),
+    'topology_uuid': {'uuid': DEFAULT_TOPOLOGY_UUID},
+}
+TOPOLOGY = {
+    'topology_id': copy.deepcopy(TOPOLOGY_ID),
+    'device_ids': [],
+    'link_ids': [],
+}
+
+DEVICE1_UUID = 'DEV1'
+DEVICE1_ID = {'device_uuid': {'uuid': DEVICE1_UUID}}
+DEVICE1 = {
+    'device_id': copy.deepcopy(DEVICE1_ID),
+    'device_type': 'packet-router',
+    'device_config': {'config_rules': [
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-1/value',
+         'resource_value': 'value1'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-2/value',
+         'resource_value': 'value2'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-3/value',
+         'resource_value': 'value3'},
+    ]},
+    'device_operational_status': DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED,
+    'device_drivers': [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_P4],
+    'device_endpoints': [
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE1_ID),
+            'endpoint_uuid': {'uuid': 'EP1'},
+        }, 'endpoint_type': 'port-packet-100G'},
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE1_ID),
+            'endpoint_uuid': {'uuid': 'EP2'},
+        }, 'endpoint_type': 'port-packet-100G'},
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE1_ID),
+            'endpoint_uuid': {'uuid': 'EP3'},
+        }, 'endpoint_type': 'port-packet-100G'},
+    ],
+}
+
+DEVICE2_UUID = 'DEV2'
+DEVICE2_ID = {'device_uuid': {'uuid': DEVICE2_UUID}}
+DEVICE2 = {
+    'device_id': copy.deepcopy(DEVICE2_ID),
+    'device_type': 'packet-router',
+    'device_config': {'config_rules': [
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-1/value',
+         'resource_value': 'value4'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-2/value',
+         'resource_value': 'value5'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'device/resource-3/value',
+         'resource_value': 'value6'},
+    ]},
+    'device_operational_status': DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED,
+    'device_drivers': [DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_P4],
+    'device_endpoints': [
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE2_ID),
+            'endpoint_uuid': {'uuid': 'EP1'},
+        }, 'endpoint_type': 'port-packet-100G'},
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE2_ID),
+            'endpoint_uuid': {'uuid': 'EP2'},
+        }, 'endpoint_type': 'port-packet-100G'},
+        {'endpoint_id': {
+            'topology_id': copy.deepcopy(TOPOLOGY_ID),
+            'device_id': copy.deepcopy(DEVICE2_ID),
+            'endpoint_uuid': {'uuid': 'EP3'},
+        }, 'endpoint_type': 'port-packet-100G'},
+    ],
+}
+
+LINK_UUID = 'DEV1/EP2 ==> DEV2/EP1'
+LINK_ID = {'link_uuid': {'uuid': LINK_UUID}}
+LINK = {
+    'link_id': copy.deepcopy(LINK_ID),
+    'link_endpoint_ids' : [
+        {'topology_id': copy.deepcopy(TOPOLOGY_ID),
+         'device_id': copy.deepcopy(DEVICE1_ID),
+         'endpoint_uuid': {'uuid' : 'EP2'}},
+        {'topology_id': copy.deepcopy(TOPOLOGY_ID),
+         'device_id': copy.deepcopy(DEVICE2_ID),
+         'endpoint_uuid': {'uuid' : 'EP1'}},
+    ]
+}
+
+SERVICE_UUID = 'SVC:DEV1/EP2-DEV2/EP1'
+SERVICE_ID = {
+    'context_id': copy.deepcopy(CONTEXT_ID),
+    'service_uuid': {'uuid': SERVICE_UUID},
+}
+SERVICE = {
+    'service_id': copy.deepcopy(SERVICE_ID),
+    'service_type': ServiceTypeEnum.SERVICETYPE_L3NM,
+    'service_endpoint_ids' : [
+        {'topology_id': copy.deepcopy(TOPOLOGY_ID),
+         'device_id': copy.deepcopy(DEVICE1_ID),
+         'endpoint_uuid': {'uuid' : 'EP2'}},
+        {'topology_id': copy.deepcopy(TOPOLOGY_ID),
+         'device_id': copy.deepcopy(DEVICE2_ID),
+         'endpoint_uuid': {'uuid' : 'EP1'}},
+    ],
+    'service_constraints': [
+        {'constraint_type': 'latency_ms', 'constraint_value': '15.2'},
+        {'constraint_type': 'jitter_us', 'constraint_value': '1.2'},
+    ],
+    'service_status': {'service_status': ServiceStatusEnum.SERVICESTATUS_ACTIVE},
+    'service_config': {'config_rules': [
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'service/resource-1/value',
+         'resource_value': 'value7'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'service/resource-2/value',
+         'resource_value': 'value8'},
+        {'action': ConfigActionEnum.CONFIGACTION_SET,
+         'resource_key': 'service/resource-3/value',
+         'resource_value': 'value9'},
+    ]},
+}

src/context/tests/test_unitary_fast_hasher.py

+import logging, pytest
+from context.service.database.Tools import (
+    FASTHASHER_DATA_ACCEPTED_FORMAT, FASTHASHER_ITEM_ACCEPTED_FORMAT, fast_hasher)
+
+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(logging.DEBUG)
+
+def test_fast_hasher():
+    with pytest.raises(TypeError) as e:
+        fast_hasher(27)
+    assert str(e.value) == "data(27) must be " + FASTHASHER_DATA_ACCEPTED_FORMAT + ", found <class 'int'>"
+
+    with pytest.raises(TypeError) as e:
+        fast_hasher({27})
+    assert str(e.value) == "data({27}) must be " + FASTHASHER_DATA_ACCEPTED_FORMAT + ", found <class 'set'>"
+
+    with pytest.raises(TypeError) as e:
+        fast_hasher({'27'})
+    assert str(e.value) == "data({'27'}) must be " + FASTHASHER_DATA_ACCEPTED_FORMAT + ", found <class 'set'>"
+
+    with pytest.raises(TypeError) as e:
+        fast_hasher([27])
+    assert str(e.value) == "data[0](27) must be " + FASTHASHER_ITEM_ACCEPTED_FORMAT + ", found <class 'int'>"
+
+    fast_hasher('hello-world')
+    fast_hasher('hello-world'.encode('UTF-8'))
+    fast_hasher(['hello', 'world'])
+    fast_hasher(('hello', 'world'))
+    fast_hasher(['hello'.encode('UTF-8'), 'world'.encode('UTF-8')])
+    fast_hasher(('hello'.encode('UTF-8'), 'world'.encode('UTF-8')))

src/context/tests/test_unitary_rest.py

+import logging, pytest, requests, time
+from google.protobuf.json_format import MessageToDict
+from common.orm.Database import Database
+from common.orm.Factory import get_database_backend, BackendEnum
+from context.proto.context_pb2 import Context, Topology
+from context.Config import RESTAPI_SERVICE_PORT, RESTAPI_BASE_URL
+from context.service.rest_server.Server import Server
+from context.service.rest_server.resources.Context import Context
+#from .populate_database import populate_example
+
+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(logging.DEBUG)
+
+RESTAPI_PORT = 10000 + RESTAPI_SERVICE_PORT # avoid privileged ports
+
+SCENARIOS = [
+    (BackendEnum.INMEMORY, {}),
+    #(BackendEnum.REDIS,    {
+    #    'REDIS_SERVICE_HOST': '10.1.7.194',
+    #    'REDIS_SERVICE_PORT': 30283,
+    #    'REDIS_DATABASE_ID': 0,
+    #}),
+]
+
+@pytest.fixture(scope='session', ids=[str(scenario[0].value) for scenario in SCENARIOS], params=SCENARIOS)
+def context_database(request):
+    backend,settings = request.param
+    LOGGER.info('Running fixture with backend={}, settings={}...'.format(str(backend), str(settings)))
+    database_backend = get_database_backend(backend=backend, **settings)
+    _database = Database(database_backend)
+    return _database
+
+@pytest.fixture(scope='session')
+def context_service_rest(context_database : Database): # pylint: disable=redefined-outer-name
+    _rest_server = Server(port=RESTAPI_PORT, base_url=RESTAPI_BASE_URL)
+    _rest_server.add_resource(Context, '/context', endpoint='api.context', resource_class_args=(context_database,))
+    _rest_server.start()
+    time.sleep(1) # bring time for the server to start
+    yield _rest_server
+    _rest_server.shutdown()
+    _rest_server.join()
+
+def test_get_topology_completed_rest_api(context_service_rest : Server): # pylint: disable=redefined-outer-name
+    # should work
+    request_url = 'http://127.0.0.1:{}{}/context'.format(RESTAPI_PORT, RESTAPI_BASE_URL)
+    LOGGER.warning('Request: GET {}'.format(str(request_url)))
+    reply = requests.get(request_url)
+    LOGGER.warning('Reply: {}'.format(str(reply.text)))
+    assert reply.status_code == 200, 'Reply failed with code {}'.format(reply.status_code)
+    json_reply = reply.json()
+    topology = MessageToDict(
+        Topology(**json_reply['topologies'][0]),
+        including_default_value_fields=True, preserving_proto_field_name=True,
+        use_integers_for_enums=False)
+    validate_topology(topology)
+    validate_topology_has_devices(topology)
+    validate_topology_has_links(topology)