Commit 5c34ea77 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Several changes (intermediate backup): 

Proto:
- Minor corrections in context.proto

Common:
- Minor corrections in common.orm.backend
- Improved definition of orm.Model class with get_instances and get_references
- Extended ORM unitary tests with get_instances and get_references

Context:
- Implemented missing methods in ContextClient
- Started to implement new Context data models using ORM.
- ContextModel functional using ORM and gRPC.
- Arranged old code file placing
parent 4e60bc00

proto/context.proto

+3 −2
Original line number Diff line number Diff line
@@ -64,8 +64,9 @@ message ContextId { 


message Context {

  ContextId context_id = 1;

  repeated Topology topology = 2;

  TeraFlowController controller = 3;

  repeated TopologyId topology_ids = 2;

  repeated ServiceId service_ids = 3;

  TeraFlowController controller = 4;

}



message ContextIdList {

run_local_tests.sh

+1 −2
Original line number Diff line number Diff line
@@ -22,8 +22,7 @@ coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ 
#    centralizedcybersecurity/tests/test_unitary.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    context/tests/test_unitary_orm_context_inmemory.py \

#    context/tests/test_unitary.py

    context/tests/test_unitary.py



#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    device/tests/test_unitary_driverapi.py \

src/common/orm/backend/inmemory/InMemoryBackend.py

+3 −3
Original line number Diff line number Diff line
@@ -70,7 +70,7 @@ class InMemoryBackend(_Backend): 
        str_key = key_to_str(key)

        with self._lock:

            container = get_dict(self._keys, str_key)

            if container is None: return None

            if container is None: return {}

            if len(fields) == 0: fields = container.keys()

            return copy.deepcopy({

                field_name : field_value for field_name,field_value in container.items() if field_name in fields
@@ -97,7 +97,7 @@ class InMemoryBackend(_Backend): 
        str_key = key_to_str(key)

        with self._lock:

            container = get_list(self._keys, str_key)

            if container is None: return None

            if container is None: return []

            return copy.deepcopy(container)



    def list_push_last(self, key : List[str], item : str) -> None:
@@ -129,7 +129,7 @@ class InMemoryBackend(_Backend): 
        str_key = key_to_str(key)

        with self._lock:

            container = get_set(self._keys, str_key)

            if container is None: return None

            if container is None: return {}

            return copy.deepcopy(container)



    def set_remove(self, key : List[str], item : str) -> None:

src/common/orm/model/Model.py

+101 −26
Original line number Diff line number Diff line 
from __future__ import annotations

import logging, re

from typing import Any, Dict, List, Mapping, Optional, Set, Tuple

from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union

from common.orm.Database import Database

from common.orm.backend.Tools import key_to_str

from common.orm.fields.ForeignKeyField import ForeignKeyField

from common.type_checkers.Checkers import chk_issubclass

from ..Exceptions import ConstraintException, MutexException

from ..fields.Field import Field

from ..fields.PrimaryKeyField import PrimaryKeyField
@@ -44,7 +45,43 @@ class MetaModel(type): 
        setattr(cls_obj, '_field_names_set', set(field_names))

        return cls_obj



KEYWORD_INSTANCES  = 'instances'

KEYWORD_LOCK       = 'lock'

KEYWORD_REFERENCES = 'references'

KEYWORD_STORED     = '_stored'



class Model(metaclass=MetaModel):

    @classmethod

    def get_backend_key_instances(cls) -> str:

        return key_to_str(['{:s}'.format(cls.__name__), KEYWORD_INSTANCES])



    @classmethod

    def get_backend_key_instance(cls, primary_key : str) -> str:

        return '{:s}[{:s}]'.format(cls.__name__, primary_key)



    @classmethod

    def get_backend_key_references(cls, primary_key : str) -> str:

        match = re.match(r'^[a-zA-Z0-9\_]+\[([^\]]*)\]', primary_key)

        if not match: primary_key = cls.get_backend_key_instance(primary_key)

        return key_to_str([primary_key, KEYWORD_REFERENCES])



    @staticmethod

    def get_backend_key_lock(backend_key : str) -> str:

        if backend_key.endswith(KEYWORD_LOCK): return backend_key

        return key_to_str([backend_key, KEYWORD_LOCK])



    @staticmethod

    def get_backend_key_locks(backend_keys : List[str]) -> List[str]:

        return [Model.get_backend_key_lock(backend_key) for backend_key in backend_keys]



    @classmethod

    def backend_key__to__instance_key(cls, backend_key : str) -> str:

        class_name = cls.__name__

        if backend_key.startswith(class_name):

            match = re.match(r'^{:s}\[([^\]]*)\]'.format(class_name), backend_key)

            if match: return match.group(1)

        return backend_key



    def __init__(self, database : Database, primary_key : str, auto_load : bool = True) -> None:

        if not isinstance(database, Database):

            str_class_path = '{}.{}'.format(Database.__module__, Database.__name__)
@@ -54,31 +91,33 @@ class Model(metaclass=MetaModel): 
        pk_field_name = self._pk_field_name # pylint: disable=no-member

        pk_field_instance : 'PrimaryKeyField' = getattr(self._model_class, pk_field_name)

        primary_key = pk_field_instance.validate(primary_key)

        if primary_key.startswith(self._class_name):

            match = re.match(r'^{:s}\[([^\]]*)\]'.format(self._class_name), primary_key)

            if match: primary_key = match.group(1)

        primary_key = self.backend_key__to__instance_key(primary_key)

        setattr(self, pk_field_name, primary_key)

        self._database = database

        self._backend = database.backend

        self._instance_key : str = '{:s}[{:s}]'.format(self._class_name, primary_key)

        self._references_key : str = key_to_str([self._instance_key, 'references'])

        self._instance_key : str = self.get_backend_key_instance(primary_key)

        self._instances_key : str = self.get_backend_key_instances()

        self._references_key : str = self.get_backend_key_references(primary_key)

        self._owner_key : Optional[str] = None

        if auto_load: self.load()



    @property

    def database(self) -> Database: return self._database



    @property

    def instance_key(self) -> str: return self._instance_key



    def lock(self, extra_keys : List[List[str]] = []):

        lock_keys = [self._instance_key, self._references_key] + extra_keys

        lock_keys = [key_to_str([lock_key, 'lock']) for lock_key in lock_keys]

        lock_keys = Model.get_backend_key_locks(

            [self._instance_key, self._instances_key, self._references_key] + extra_keys)

        acquired,self._owner_key = self._backend.lock(lock_keys, owner_key=self._owner_key)

        if acquired: return

        raise MutexException('Unable to lock keys {:s} using owner_key {:s}'.format(

            str(lock_keys), str(self._owner_key)))



    def unlock(self, extra_keys : List[List[str]] = []):

        lock_keys = [self._instance_key, self._references_key] + extra_keys

        lock_keys = [key_to_str([lock_key, 'lock']) for lock_key in lock_keys]

        lock_keys = Model.get_backend_key_locks(

            [self._instance_key, self._instances_key, self._references_key] + extra_keys)

        released = self._backend.unlock(lock_keys, self._owner_key)

        if released: return

        raise MutexException('Unable to unlock keys {:s} using owner_key {:s}'.format(
@@ -99,7 +138,7 @@ class Model(metaclass=MetaModel): 
                field_instance : 'Field' = getattr(self._model_class, field_name)

                field_value = field_instance.deserialize(raw_field_value)

                if isinstance(field_instance, ForeignKeyField):

                    setattr(self, field_name + '_stored', field_value)

                    setattr(self, field_name + KEYWORD_STORED, field_value)

                    field_value = field_instance.foreign_model(self._database, field_value, auto_load=True)

                setattr(self, field_name, field_value)

        finally:
@@ -117,10 +156,10 @@ class Model(metaclass=MetaModel): 
            if (serialized_field_value is None) and (not field_instance.required): continue

            if isinstance(field_instance, ForeignKeyField):

                foreign_reference = '{:s}:{:s}'.format(self._instance_key, field_name)

                field_value_stored = getattr(self, field_name + '_stored', None)

                field_value_stored = getattr(self, field_name + KEYWORD_STORED, None)

                if field_value_stored is not None:

                    foreign_removals[key_to_str([field_value_stored, 'references'])] = foreign_reference

                foreign_additions[key_to_str([serialized_field_value, 'references'])] = foreign_reference

                    foreign_removals[self.get_backend_key_references(field_value_stored)] = foreign_reference

                foreign_additions[self.get_backend_key_references(serialized_field_value)] = foreign_reference

                required_keys.add(serialized_field_value)

            attributes[field_name] = serialized_field_value
@@ -131,14 +170,16 @@ class Model(metaclass=MetaModel): 
        try:

            self.lock(extra_keys=extra_keys)



            not_exists = []

            for required_key in required_keys:

                if self._backend.exists(required_key): continue

                not_exists.append('{:s}'.format(str(required_key)))

            not_exists = [

                str(required_key)

                for required_key in required_keys

                if not self._backend.exists(required_key)]

            if len(not_exists) > 0:

                raise ConstraintException('Required Keys ({:s}) does not exist'.format(', '.join(sorted(not_exists))))



            self._backend.dict_update(self._instance_key, attributes)

            self._backend.set_add(self._instances_key, self._instance_key)



            for serialized_field_value,foreign_reference in foreign_removals.items():

                self._backend.set_remove(serialized_field_value, foreign_reference)
@@ -148,7 +189,7 @@ class Model(metaclass=MetaModel): 
            self.unlock(extra_keys=extra_keys)



        for serialized_field_value,foreign_reference in foreign_additions.items():

            setattr(self, (foreign_reference.rsplit(':', 1)[-1]) + '_stored', field_value_stored)

            setattr(self, (foreign_reference.rsplit(':', 1)[-1]) + KEYWORD_STORED, field_value_stored)



    def delete(self) -> None:

        foreign_removals : Dict[str, str] = {}
@@ -156,9 +197,9 @@ class Model(metaclass=MetaModel): 
            field_instance : 'Field' = getattr(self._model_class, field_name)

            if not isinstance(field_instance, ForeignKeyField): continue

            foreign_reference = '{:s}:{:s}'.format(self._instance_key, field_name)

            field_value_stored = getattr(self, field_name + '_stored', None)

            field_value_stored = getattr(self, field_name + KEYWORD_STORED, None)

            if field_value_stored is None: continue

            foreign_removals[key_to_str([field_value_stored, 'references'])] = foreign_reference

            foreign_removals[self.get_backend_key_references(field_value_stored)] = foreign_reference



        extra_keys = []

        extra_keys.extend(list(foreign_removals.keys()))
@@ -171,21 +212,55 @@ class Model(metaclass=MetaModel): 
                raise ConstraintException('Instance is used by Keys ({:s})'.format(', '.join(sorted(references))))



            self._backend.delete(self._instance_key)

            self._backend.set_remove(self._instances_key, self._instance_key)



            for serialized_field_value,foreign_reference in foreign_removals.items():

                self._backend.set_remove(serialized_field_value, foreign_reference)

        finally:

            self.unlock(extra_keys=extra_keys)



    def references(self) -> Set[Tuple[str, str]]:

    def references(

        self, filter_by_models : Optional[Union[type, List[type], Set[type]]] = None) -> Set[Tuple[str, str]]:

        try:

            self.lock()

            if self._backend.exists(self._references_key):

            if not self._backend.exists(self._references_key): return {}

            references = self._backend.set_get_all(self._references_key)

            if filter_by_models is None:

                pass

            elif isinstance(filter_by_models, (list, set)):

                filter_by_models = {type(chk_issubclass(model, Model)).__name__ for model in filter_by_models}

            elif issubclass(filter_by_models, Model):

                filter_by_models = {type(chk_issubclass(filter_by_models, Model)).__name__}

            else:

                msg = 'filter_by_models({:s}) unsupported. Expected a type or a list/set of types. Optionally, keep '\

                      'it as None to retrieve all the references pointing to this instance.'

                raise AttributeError(msg.format(str(filter_by_models)))

            if filter_by_models:

                references = filter(lambda instance_key: instance_key.split('[', 1)[0] in filter_by_models, references)

            return {tuple(reference.rsplit(':', 1)) for reference in references}

            return {}

        finally:

            self.unlock()



    @classmethod

    def get_primary_keys(cls, database : Database):

        backend = database.backend

        key_model_instances = cls.get_backend_key_instances()

        key_model_instances_lock = cls.get_backend_key_lock(key_model_instances)



        acquired,owner_key = backend.lock(key_model_instances_lock)

        if not acquired:

            raise MutexException('Unable to lock keys {:s}'.format(

                str(key_model_instances_lock)))



        instance_keys = backend.set_get_all(key_model_instances)



        released = backend.unlock(key_model_instances_lock, owner_key)

        if not released:

            raise MutexException('Unable to unlock keys {:s} using owner_key {:s}'.format(

                str(key_model_instances_lock), str(owner_key)))



        return instance_keys



    def dump_id(self) -> Dict:

        raise NotImplementedError()

src/common/orm/tests/test_unitary.py

+75 −35
Original line number Diff line number Diff line
@@ -85,6 +85,18 @@ def test_model_with_primarykey(): 
        active = BooleanField()

        gender = EnumeratedField(GenderEnum)



    backend_key_instances  = TestModel.get_backend_key_instances()

    backend_key_instance   = TestModel.get_backend_key_instance('pk')

    backend_key_references = TestModel.get_backend_key_references('pk')



    assert backend_key_instances  == 'TestModel/instances'

    assert backend_key_instance   == 'TestModel[pk]'

    assert backend_key_references == 'TestModel[pk]/references'



    assert TestModel.get_backend_key_lock(backend_key_instances ) == 'TestModel/instances/lock'

    assert TestModel.get_backend_key_lock(backend_key_instance  ) == 'TestModel[pk]/lock'

    assert TestModel.get_backend_key_lock(backend_key_references) == 'TestModel[pk]/references/lock'



    with pytest.raises(ValueError) as e:

        TestModel(database, None)

    assert str(e.value) == 'pk(None) is required. It cannot be None.'
@@ -248,8 +260,11 @@ def test_model_database_operations(): 
    obj.save()



    db_entries = database.dump()

    assert len(db_entries) == 1

    assert len(db_entries) == 2

    assert db_entries[0] == (

        'set', 'TestModel/instances',

        "{'TestModel[valid-pk]'}")

    assert db_entries[1] == (

        'dict', 'TestModel[valid-pk]',

        "{'active': 'True', 'age': '37', 'gender': 'MALE', 'name': 'John Smith', 'pk': 'valid-pk', "\

        "'salary': '5023.52'}")
@@ -270,7 +285,20 @@ def test_model_database_operations(): 
    assert len(database.dump()) == 0



    obj2.save()

    assert len(database.dump()) == 1



    db_entries = database.dump()

    LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries)))

    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry))

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 2

    assert db_entries[0] == (

        'set', 'TestModel/instances',

        "{'TestModel[valid-pk]'}")

    assert db_entries[1] == (

        'dict', 'TestModel[valid-pk]',

        "{'active': 'True', 'age': '37', 'gender': 'MALE', 'name': 'John Smith', 'pk': 'valid-pk', "\

        "'salary': '5023.52'}")



    database.clear_all()

    assert len(database.dump()) == 0
@@ -462,28 +490,34 @@ def test_model_foreignkeys(): 
        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry))

    LOGGER.info('-----------------------------------------------------------')



    assert len(db_entries) == 10

    assert db_entries[0] == ('dict', "Member[brad]",

    assert len(db_entries) == 13

    assert db_entries[ 0] == ('set', "Member/instances",

                              "{'Member[brad]', 'Member[jane]', 'Member[john]'}")

    assert db_entries[ 1] == ('dict', "Member[brad]",

                              "{'gender': 'MALE', 'name': 'Brad', 'pk': 'brad', 'team': 'Team[admin]'}")

    assert db_entries[1] == ('dict', "Member[jane]",

    assert db_entries[ 2] == ('dict', "Member[jane]",

                              "{'gender': 'FEMALE', 'name': 'Jane', 'pk': 'jane', 'place': 'Workplace[mad]', "\

                              "'team': 'Team[dev-ops]'}")

    assert db_entries[2] == ('dict', "Member[john]",

    assert db_entries[ 3] == ('dict', "Member[john]",

                              "{'gender': 'MALE', 'name': 'John', 'pk': 'john', 'place': 'Workplace[mad]', "\

                              "'team': 'Team[dev-ops]'}")

    assert db_entries[3] == ('dict', "Team[admin]",

    assert db_entries[ 4] == ('set', "Team/instances",

                              "{'Team[admin]', 'Team[dev-ops]'}")

    assert db_entries[ 5] == ('dict', "Team[admin]",

                              "{'name': 'Admin', 'pk': 'admin'}")

    assert db_entries[4] == ('set' , "Team[admin]/references",

    assert db_entries[ 6] == ('set' , "Team[admin]/references",

                              "{'Member[brad]:team'}")

    assert db_entries[5] == ('dict', "Team[dev-ops]",

    assert db_entries[ 7] == ('dict', "Team[dev-ops]",

                              "{'name': 'Dev Ops', 'pk': 'dev-ops'}")

    assert db_entries[6] == ('set' , "Team[dev-ops]/references",

    assert db_entries[ 8] == ('set' , "Team[dev-ops]/references",

                              "{'Member[jane]:team', 'Member[john]:team'}")

    assert db_entries[7] == ('dict', "Workplace[bcn]",

    assert db_entries[ 9] == ('set', "Workplace/instances",

                              "{'Workplace[bcn]', 'Workplace[mad]'}")

    assert db_entries[10] == ('dict', "Workplace[bcn]",

                              "{'name': 'Barcelona', 'pk': 'bcn'}")

    assert db_entries[8] == ('dict', "Workplace[mad]",

    assert db_entries[11] == ('dict', "Workplace[mad]",

                              "{'name': 'Madrid', 'pk': 'mad'}")

    assert db_entries[9] == ('set' , "Workplace[mad]/references",

    assert db_entries[12] == ('set' , "Workplace[mad]/references",

                              "{'Member[jane]:place', 'Member[john]:place'}")



    Member(database, member_john.pk).delete()
@@ -494,23 +528,29 @@ def test_model_foreignkeys(): 
        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry))

    LOGGER.info('-----------------------------------------------------------')



    assert len(db_entries) == 9

    assert db_entries[ 0] == ('dict', 'Member[brad]',

    assert len(db_entries) == 12

    assert db_entries[ 0] == ('set', "Member/instances",

                              "{'Member[brad]', 'Member[jane]'}")

    assert db_entries[ 1] == ('dict', 'Member[brad]',

                              "{'gender': 'MALE', 'name': 'Brad', 'pk': 'brad', 'team': 'Team[admin]'}")

    assert db_entries[ 1] == ('dict', 'Member[jane]',

    assert db_entries[ 2] == ('dict', 'Member[jane]',

                              "{'gender': 'FEMALE', 'name': 'Jane', 'pk': 'jane', 'place': 'Workplace[mad]', "\

                              "'team': 'Team[dev-ops]'}")

    assert db_entries[ 2] == ('dict', 'Team[admin]',

    assert db_entries[ 3] == ('set', "Team/instances",

                              "{'Team[admin]', 'Team[dev-ops]'}")

    assert db_entries[ 4] == ('dict', 'Team[admin]',

                              "{'name': 'Admin', 'pk': 'admin'}")

    assert db_entries[ 3] == ('set',  'Team[admin]/references',

    assert db_entries[ 5] == ('set',  'Team[admin]/references',

                              "{'Member[brad]:team'}")

    assert db_entries[ 4] == ('dict', 'Team[dev-ops]',

    assert db_entries[ 6] == ('dict', 'Team[dev-ops]',

                              "{'name': 'Dev Ops', 'pk': 'dev-ops'}")

    assert db_entries[ 5] == ('set',  'Team[dev-ops]/references',

    assert db_entries[ 7] == ('set',  'Team[dev-ops]/references',

                              "{'Member[jane]:team'}")

    assert db_entries[ 6] == ('dict', 'Workplace[bcn]',

    assert db_entries[ 8] == ('set', "Workplace/instances",

                              "{'Workplace[bcn]', 'Workplace[mad]'}")

    assert db_entries[ 9] == ('dict', 'Workplace[bcn]',

                              "{'name': 'Barcelona', 'pk': 'bcn'}")

    assert db_entries[ 7] == ('dict', 'Workplace[mad]',

    assert db_entries[10] == ('dict', 'Workplace[mad]',

                              "{'name': 'Madrid', 'pk': 'mad'}")

    assert db_entries[ 8] == ('set',  'Workplace[mad]/references',

    assert db_entries[11] == ('set',  'Workplace[mad]/references',

                              "{'Member[jane]:place'}")