Commit 39474919 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Several Changes: 

Common:
- Renamed common.metrics to common.rpc_method_wrapper
- Moved ServiceException to common.rpc_method_wrapper
- Implemented High-Level ORM methods to simplify Database handling

Context:
- Added flag to auto-populate fake data for testing purposes
- Added fake objects for testing purposes
- Minor corrections in ORM Database Models
- Reimplemented gRPC Servicer methods using HighLevel ORM helper methods
- Implemented complete unit testing of gRPC Servicer
- Added Message Broker to enable distribution of database events
- Implemented gRPC GetEvents methods
- Modified default config values to support larger number of persistent connections for events streaming
- Reimplemented Context REST Resource to re-use gRPC Servicer and prevent duplicating code

Automated Tests:
- Reactivated common and context unit tests
parent 5a10c6f5

run_local_tests.sh

+3 −3
Original line number Diff line number Diff line
@@ -15,14 +15,14 @@ cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/teraflow/controller+$PRO 
rm -f $COVERAGEFILE



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    common/metrics/tests/test_unitary.py \

    common/orm/tests/test_unitary.py \

    common/message_broker/tests/test_unitary.py

    common/message_broker/tests/test_unitary.py \

    common/rpc_method_wrapper/tests/test_unitary.py



#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    centralizedcybersecurity/tests/test_unitary.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose --maxfail=1\

    context/tests/test_unitary_fast_hasher.py \

    context/tests/test_unitary_grpc.py \

    #context/tests/test_unitary_rest.py

src/common/exceptions/ServiceException.py

deleted100644 → 0
+0 −7
Original line number Diff line number Diff line 
import grpc



class ServiceException(Exception):

    def __init__(self, code : grpc.StatusCode, details : str) -> None:

        self.code = code

        self.details = details

        super().__init__(self.details)

src/common/metrics/tests/__init__.py

deleted100644 → 0
+0 −0

Empty file deleted.

src/common/orm/HighLevel.py

0 → 100644
+70 −0
Original line number Diff line number Diff line 
from typing import Any, Dict, List, Optional, Set, Tuple

from common.rpc_method_wrapper.ServiceExceptions import NotFoundException

from common.orm.Database import Database

from common.orm.backend.Tools import key_to_str

from common.orm.fields.ForeignKeyField import ForeignKeyField

from common.orm.model.Model import Model, MetaModel



def get_all_objects(database : Database, model_class : MetaModel) -> List[Model]:

    db_pks = sorted(list(model_class.get_primary_keys(database)))

    return [model_class(database, db_pk) for db_pk in db_pks]



def get_object(

    database : Database, model_class : Model, key_parts : List[str], raise_if_not_found : bool = True

    ) -> Optional[Model]:



    str_key = key_to_str(key_parts)

    db_object = model_class(database, str_key, auto_load=False)

    found = db_object.load()

    if found: return db_object

    if raise_if_not_found: raise NotFoundException(model_class.__name__.replace('Model', ''), str_key)

    return None



def get_related_objects(

    source_instance : Model, reference_model_class : MetaModel, navigation_field_name : str = None) -> Set[Model]:



    database = source_instance.database

    db_target_instances = set()



    if navigation_field_name is not None:

        navigation_fk_field : Optional[ForeignKeyField] = getattr(reference_model_class, navigation_field_name, None)

        if navigation_fk_field is None or not isinstance(navigation_fk_field, ForeignKeyField):

            msg = 'navigation_field_name({:s}) must be a ForeignKeyField in reference_model_class({:s})'

            raise AttributeError(msg.format(navigation_field_name, reference_model_class.__name__))

        target_model_class = navigation_fk_field.foreign_model



    for db_reference_pk,_ in source_instance.references(reference_model_class):

        db_reference = reference_model_class(database, db_reference_pk)

        if navigation_field_name is not None:

            target_fk_field = getattr(db_reference, navigation_field_name, None)

            if target_fk_field is None: continue

            db_reference = target_model_class(database, target_fk_field)

        db_target_instances.add(db_reference)

    return db_target_instances



def update_or_create_object(

    database : Database, model_class : Model, key_parts : List[str], attributes : Dict[str, Any]

    ) -> Tuple[Model, bool]:



    str_key = key_to_str(key_parts)

    db_object : Model = model_class(database, str_key, auto_load=False)

    found = db_object.load()

    for attr_name, attr_value in attributes.items():

        setattr(db_object, attr_name, attr_value)

    db_object.save()

    updated = found # updated if found, else created

    return db_object, updated



def get_or_create_object(

    database : Database, model_class : Model, key_parts : List[str], defaults : Dict[str, Any] = {}

    ) -> Tuple[Model, bool]:



    str_key = key_to_str(key_parts)

    db_object : Model = model_class(database, str_key, auto_load=False)

    found = db_object.load()

    if not found:

        for attr_name, attr_value in defaults.items():

            setattr(db_object, attr_name, attr_value)

        db_object.save()

    created = not found # created if not found, else loaded

    return db_object, created

src/common/orm/model/Model.py

+0 −16
Original line number Diff line number Diff line
@@ -290,19 +290,3 @@ class Model(metaclass=MetaModel): 
            for name in self._field_names_list # pylint: disable=no-member

        )

        return '{:s}({:s})'.format(self._class_name, arguments)
 


def get_related_instances(

    source_instance : Model, relation_model_class : MetaModel, relation_field_name : str) -> Set[Model]:

    navigation_fk_field : ForeignKeyField = getattr(relation_model_class, relation_field_name, None)

    if navigation_fk_field is None or not isinstance(navigation_fk_field, ForeignKeyField):

        msg = 'relation_field_name({:s}) must be a ForeignKeyField in relation_model_class({:s})'

        raise AttributeError(msg.format(relation_field_name, relation_model_class.__name__))

    target_model_class = navigation_fk_field.foreign_model

    database = source_instance.database

    db_target_instances = set()

    for db_relation_pk,_ in source_instance.references(relation_model_class):

        db_relation = relation_model_class(database, db_relation_pk)

        target_fk_field = getattr(db_relation, relation_field_name, None)

        if target_fk_field is None: continue

        db_target_instances.add(target_model_class(database, target_fk_field))

    return db_target_instances