Commit 6fc3ba23 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component: 

- pre-merge cleanup of policy persistence
parent 5f8aa907

src/context/client/ContextClient.py

+36 −33
Original line number Original line Diff line number Diff line
@@ -29,7 +29,7 @@ from common.proto.context_pb2 import ( 
    Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList)

    Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList)

from common.proto.context_pb2_grpc import ContextServiceStub

from common.proto.context_pb2_grpc import ContextServiceStub

from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub

from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub

from common.proto.policy_pb2 import (PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule)

from common.proto.policy_pb2 import PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule





LOGGER = logging.getLogger(__name__)

LOGGER = logging.getLogger(__name__)

MAX_RETRIES = 15

MAX_RETRIES = 15
@@ -59,38 +59,6 @@ class ContextClient: 
        self.stub = None

        self.stub = None

        self.policy_stub = None

        self.policy_stub = None







    @RETRY_DECORATOR

    def ListPolicyRuleIds(self, request: Empty) -> PolicyRuleIdList:

        LOGGER.debug('ListPolicyRuleIds request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRuleIds(request)

        LOGGER.debug('ListPolicyRuleIds result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def ListPolicyRules(self, request: Empty) -> PolicyRuleList:

        LOGGER.debug('ListPolicyRules request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRules(request)

        LOGGER.debug('ListPolicyRules result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def GetPolicyRule(self, request: PolicyRuleId) -> PolicyRule:

        LOGGER.info('GetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.GetPolicyRule(request)

        LOGGER.info('GetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def SetPolicyRule(self, request: PolicyRule) -> PolicyRuleId:

        LOGGER.debug('SetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.SetPolicyRule(request)

        LOGGER.debug('SetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def RemovePolicyRule(self, request: PolicyRuleId) -> Empty:

        LOGGER.debug('RemovePolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.RemovePolicyRule(request)

        LOGGER.debug('RemovePolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    @RETRY_DECORATOR

    def ListContextIds(self, request: Empty) -> ContextIdList:

    def ListContextIds(self, request: Empty) -> ContextIdList:

        LOGGER.debug('ListContextIds request: {:s}'.format(grpc_message_to_json_string(request)))

        LOGGER.debug('ListContextIds request: {:s}'.format(grpc_message_to_json_string(request)))
@@ -398,3 +366,38 @@ class ContextClient: 
        response = self.stub.GetConnectionEvents(request)

        response = self.stub.GetConnectionEvents(request)

        LOGGER.debug('GetConnectionEvents result: {:s}'.format(grpc_message_to_json_string(response)))

        LOGGER.debug('GetConnectionEvents result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

        return response



    @RETRY_DECORATOR

    def ListPolicyRuleIds(self, request: Empty) -> PolicyRuleIdList:

        LOGGER.debug('ListPolicyRuleIds request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRuleIds(request)

        LOGGER.debug('ListPolicyRuleIds result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def ListPolicyRules(self, request: Empty) -> PolicyRuleList:

        LOGGER.debug('ListPolicyRules request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRules(request)

        LOGGER.debug('ListPolicyRules result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def GetPolicyRule(self, request: PolicyRuleId) -> PolicyRule:

        LOGGER.info('GetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.GetPolicyRule(request)

        LOGGER.info('GetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def SetPolicyRule(self, request: PolicyRule) -> PolicyRuleId:

        LOGGER.debug('SetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.SetPolicyRule(request)

        LOGGER.debug('SetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def RemovePolicyRule(self, request: PolicyRuleId) -> Empty:

        LOGGER.debug('RemovePolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.RemovePolicyRule(request)

        LOGGER.debug('RemovePolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

src/context/service/database/PolicyRuleModel.py

+2 −5
Original line number Original line Diff line number Diff line
@@ -14,22 +14,19 @@ 




import logging

import logging

import json

import json

from typing import Dict, List, Optional, Tuple

from typing import Dict

from common.orm.fields.PrimaryKeyField import PrimaryKeyField

from common.orm.fields.PrimaryKeyField import PrimaryKeyField

from common.orm.fields.StringField import StringField

from common.orm.fields.StringField import StringField

from common.orm.model.Model import Model

from common.orm.model.Model import Model





LOGGER = logging.getLogger(__name__)

LOGGER = logging.getLogger(__name__)







class PolicyRuleModel(Model):

class PolicyRuleModel(Model):

    pk = PrimaryKeyField()

    pk = PrimaryKeyField()

    value = StringField(required=True, allow_empty=False)

    value = StringField(required=True, allow_empty=False)





    def dump_id(self) -> Dict:

    def dump_id(self) -> Dict:

        return {

        return {'uuid': {'uuid': self.pk}}

            "uuid": {"uuid": self.pk}

        }





    def dump(self) -> Dict:

    def dump(self) -> Dict:

        return json.loads(self.value)
 
        return json.loads(self.value)

src/context/service/grpc_server/Constants.py

+1 −3
Original line number Original line Diff line number Diff line
@@ -19,9 +19,7 @@ TOPIC_DEVICE = 'device' 
TOPIC_LINK       = 'link'

TOPIC_LINK       = 'link'

TOPIC_SERVICE    = 'service'

TOPIC_SERVICE    = 'service'

TOPIC_SLICE      = 'slice'

TOPIC_SLICE      = 'slice'

TOPIC_POLICY     = 'policy'





TOPICS = {TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_TOPOLOGY, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE,

TOPICS = {TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_TOPOLOGY, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE}

          TOPIC_POLICY}





CONSUME_TIMEOUT = 0.5 # seconds
 
CONSUME_TIMEOUT = 0.5 # seconds

src/context/service/grpc_server/ContextServiceServicerImpl.py

+13 −15
Original line number Original line Diff line number Diff line
@@ -33,6 +33,7 @@ from common.proto.context_pb2_grpc import ContextServiceServicer 
from common.proto.context_policy_pb2_grpc import ContextPolicyServiceServicer

from common.proto.context_policy_pb2_grpc import ContextPolicyServiceServicer

from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method

from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method

from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException

from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException

from common.tools.grpc.Tools import grpc_message_to_json

from context.service.database.ConfigModel import update_config

from context.service.database.ConfigModel import update_config

from context.service.database.ConnectionModel import ConnectionModel, set_path

from context.service.database.ConnectionModel import ConnectionModel, set_path

from context.service.database.ConstraintModel import set_constraints

from context.service.database.ConstraintModel import set_constraints
@@ -51,8 +52,7 @@ from context.service.database.SliceModel import SliceModel, grpc_to_enum__slice_ 
from context.service.database.TopologyModel import TopologyModel

from context.service.database.TopologyModel import TopologyModel

from .Constants import (

from .Constants import (

    CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE,

    CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE,

    TOPIC_TOPOLOGY, TOPIC_POLICY)

    TOPIC_TOPOLOGY)

from common.tools.grpc.Tools import grpc_message_to_json_string, grpc_message_to_json





LOGGER = logging.getLogger(__name__)

LOGGER = logging.getLogger(__name__)
@@ -820,7 +820,8 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
            yield ConnectionEvent(**json.loads(message.content))

            yield ConnectionEvent(**json.loads(message.content))









    # ----- Policy ----------------------------------------------------------------------------------------------------

    # ----- Policy -----------------------------------------------------------------------------------------------------



    @safe_and_metered_rpc_method(METRICS, LOGGER)

    @safe_and_metered_rpc_method(METRICS, LOGGER)

    def ListPolicyRuleIds(self, request: Empty, context: grpc.ServicerContext) -> PolicyRuleIdList:

    def ListPolicyRuleIds(self, request: Empty, context: grpc.ServicerContext) -> PolicyRuleIdList:

        with self.lock:

        with self.lock:
@@ -833,8 +834,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
        with self.lock:

        with self.lock:

            db_policy_rules: List[PolicyRuleModel] = get_all_objects(self.database, PolicyRuleModel)

            db_policy_rules: List[PolicyRuleModel] = get_all_objects(self.database, PolicyRuleModel)

            db_policy_rules = sorted(db_policy_rules, key=operator.attrgetter('pk'))

            db_policy_rules = sorted(db_policy_rules, key=operator.attrgetter('pk'))

            res = PolicyRuleList(policyRules=[db_policy_rule.dump_id() for db_policy_rule in db_policy_rules])

            return PolicyRuleList(policyRules=[db_policy_rule.dump() for db_policy_rule in db_policy_rules])

            return res





    @safe_and_metered_rpc_method(METRICS, LOGGER)

    @safe_and_metered_rpc_method(METRICS, LOGGER)

    def GetPolicyRule(self, request: PolicyRuleId, context: grpc.ServicerContext) -> PolicyRule:

    def GetPolicyRule(self, request: PolicyRuleId, context: grpc.ServicerContext) -> PolicyRule:
@@ -848,15 +848,14 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
        with self.lock:

        with self.lock:

            policy_rule_type = request.WhichOneof('policy_rule')

            policy_rule_type = request.WhichOneof('policy_rule')

            policy_rule_json = grpc_message_to_json(request)

            policy_rule_json = grpc_message_to_json(request)

            policy_rule_uuid = policy_rule_json[policy_rule_type]["policyRuleBasic"]["policyRuleId"]["uuid"]["uuid"]

            policy_rule_uuid = policy_rule_json[policy_rule_type]['policyRuleBasic']['policyRuleId']['uuid']['uuid']

            result: Tuple[PolicyRuleModel, bool] = update_or_create_object(

            result: Tuple[PolicyRuleModel, bool] = update_or_create_object(

                self.database, PolicyRuleModel, policy_rule_uuid, {"value": json.dumps(policy_rule_json)})

                self.database, PolicyRuleModel, policy_rule_uuid, {'value': json.dumps(policy_rule_json)})

            db_policy, updated = result

            db_policy, updated = result # pylint: disable=unused-variable





            event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

            #event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

            dict_policy_id = db_policy.dump_id()

            dict_policy_id = db_policy.dump_id()

            notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            #notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})



            return PolicyRuleId(**dict_policy_id)

            return PolicyRuleId(**dict_policy_id)





    @safe_and_metered_rpc_method(METRICS, LOGGER)

    @safe_and_metered_rpc_method(METRICS, LOGGER)
@@ -865,11 +864,10 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
            policy_uuid = request.uuid.uuid

            policy_uuid = request.uuid.uuid

            db_policy = PolicyRuleModel(self.database, policy_uuid, auto_load=False)

            db_policy = PolicyRuleModel(self.database, policy_uuid, auto_load=False)

            found = db_policy.load()

            found = db_policy.load()

            if not found:

            if not found: return Empty()

                return Empty()





            dict_policy_id = db_policy.dump_id()

            dict_policy_id = db_policy.dump_id()

            db_policy.delete()

            db_policy.delete()

            event_type = EventTypeEnum.EVENTTYPE_REMOVE

            #event_type = EventTypeEnum.EVENTTYPE_REMOVE

            notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            #notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            return Empty()
 
            return Empty()

src/context/service/rest_server/Resources.py

+22 −0
Original line number Original line Diff line number Diff line
@@ -17,6 +17,7 @@ from flask.json import jsonify 
from flask_restful import Resource

from flask_restful import Resource

from common.orm.Database import Database

from common.orm.Database import Database

from common.proto.context_pb2 import ConnectionId, ContextId, DeviceId, Empty, LinkId, ServiceId, SliceId, TopologyId

from common.proto.context_pb2 import ConnectionId, ContextId, DeviceId, Empty, LinkId, ServiceId, SliceId, TopologyId

from common.proto.policy_pb2 import PolicyRuleId

from common.tools.grpc.Tools import grpc_message_to_json

from common.tools.grpc.Tools import grpc_message_to_json

from context.service.grpc_server.ContextServiceServicerImpl import ContextServiceServicerImpl

from context.service.grpc_server.ContextServiceServicerImpl import ContextServiceServicerImpl
@@ -61,6 +62,11 @@ def grpc_topology_id(context_uuid, topology_uuid): 
        'topology_uuid': {'uuid': topology_uuid}

        'topology_uuid': {'uuid': topology_uuid}

    })

    })





def grpc_policy_rule_id(policy_rule_uuid):

    return PolicyRuleId(**{

        'uuid': {'uuid': policy_rule_uuid}

    })



class _Resource(Resource):

class _Resource(Resource):

    def __init__(self, database : Database) -> None:

    def __init__(self, database : Database) -> None:

        super().__init__()

        super().__init__()
@@ -151,6 +157,18 @@ class Connection(_Resource): 
    def get(self, connection_uuid : str):

    def get(self, connection_uuid : str):

        return format_grpc_to_json(self.servicer.GetConnection(grpc_connection_id(connection_uuid), None))

        return format_grpc_to_json(self.servicer.GetConnection(grpc_connection_id(connection_uuid), None))





class PolicyRuleIds(_Resource):

    def get(self):

        return format_grpc_to_json(self.servicer.ListPolicyRuleIds(Empty(), None))



class PolicyRules(_Resource):

    def get(self):

        return format_grpc_to_json(self.servicer.ListPolicyRules(Empty(), None))



class PolicyRule(_Resource):

    def get(self, policy_rule_uuid : str):

        return format_grpc_to_json(self.servicer.GetPolicyRule(grpc_policy_rule_id(policy_rule_uuid), None))



class DumpText(Resource):

class DumpText(Resource):

    def __init__(self, database : Database) -> None:

    def __init__(self, database : Database) -> None:

        super().__init__()

        super().__init__()
@@ -219,6 +237,10 @@ RESOURCES = [ 
    ('api.connections',    Connections,   '/context/<string:context_uuid>/service/<path:service_uuid>/connections'),

    ('api.connections',    Connections,   '/context/<string:context_uuid>/service/<path:service_uuid>/connections'),

    ('api.connection',     Connection,    '/connection/<path:connection_uuid>'),

    ('api.connection',     Connection,    '/connection/<path:connection_uuid>'),





    ('api.policyrule_ids', PolicyRuleIds, '/policyrule_ids'),

    ('api.policyrules',    PolicyRules,   '/policyrules'),

    ('api.policyrule',     PolicyRule,    '/policyrule/<string:policyrule_uuid>'),



    ('api.dump.text',      DumpText,      '/dump/text'),

    ('api.dump.text',      DumpText,      '/dump/text'),

    ('api.dump.html',      DumpHtml,      '/dump/html'),

    ('api.dump.html',      DumpHtml,      '/dump/html'),

]
 
]