Commit 6fc3ba23 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context component: 

- pre-merge cleanup of policy persistence
parent 5f8aa907

src/context/client/ContextClient.py

+36 −33
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ from common.proto.context_pb2 import ( 
    Topology, TopologyEvent, TopologyId, TopologyIdList, TopologyList)

from common.proto.context_pb2_grpc import ContextServiceStub

from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub

from common.proto.policy_pb2 import (PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule)

from common.proto.policy_pb2 import PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule



LOGGER = logging.getLogger(__name__)

MAX_RETRIES = 15
@@ -59,38 +59,6 @@ class ContextClient: 
        self.stub = None

        self.policy_stub = None





    @RETRY_DECORATOR

    def ListPolicyRuleIds(self, request: Empty) -> PolicyRuleIdList:

        LOGGER.debug('ListPolicyRuleIds request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRuleIds(request)

        LOGGER.debug('ListPolicyRuleIds result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def ListPolicyRules(self, request: Empty) -> PolicyRuleList:

        LOGGER.debug('ListPolicyRules request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRules(request)

        LOGGER.debug('ListPolicyRules result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def GetPolicyRule(self, request: PolicyRuleId) -> PolicyRule:

        LOGGER.info('GetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.GetPolicyRule(request)

        LOGGER.info('GetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def SetPolicyRule(self, request: PolicyRule) -> PolicyRuleId:

        LOGGER.debug('SetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.SetPolicyRule(request)

        LOGGER.debug('SetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

    @RETRY_DECORATOR

    def RemovePolicyRule(self, request: PolicyRuleId) -> Empty:

        LOGGER.debug('RemovePolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.RemovePolicyRule(request)

        LOGGER.debug('RemovePolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def ListContextIds(self, request: Empty) -> ContextIdList:

        LOGGER.debug('ListContextIds request: {:s}'.format(grpc_message_to_json_string(request)))
@@ -398,3 +366,38 @@ class ContextClient: 
        response = self.stub.GetConnectionEvents(request)

        LOGGER.debug('GetConnectionEvents result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def ListPolicyRuleIds(self, request: Empty) -> PolicyRuleIdList:

        LOGGER.debug('ListPolicyRuleIds request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRuleIds(request)

        LOGGER.debug('ListPolicyRuleIds result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def ListPolicyRules(self, request: Empty) -> PolicyRuleList:

        LOGGER.debug('ListPolicyRules request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.ListPolicyRules(request)

        LOGGER.debug('ListPolicyRules result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def GetPolicyRule(self, request: PolicyRuleId) -> PolicyRule:

        LOGGER.info('GetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.GetPolicyRule(request)

        LOGGER.info('GetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def SetPolicyRule(self, request: PolicyRule) -> PolicyRuleId:

        LOGGER.debug('SetPolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.SetPolicyRule(request)

        LOGGER.debug('SetPolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response



    @RETRY_DECORATOR

    def RemovePolicyRule(self, request: PolicyRuleId) -> Empty:

        LOGGER.debug('RemovePolicyRule request: {:s}'.format(grpc_message_to_json_string(request)))

        response = self.policy_stub.RemovePolicyRule(request)

        LOGGER.debug('RemovePolicyRule result: {:s}'.format(grpc_message_to_json_string(response)))

        return response

src/context/service/database/PolicyRuleModel.py

+2 −5
Original line number Diff line number Diff line
@@ -14,22 +14,19 @@ 


import logging

import json

from typing import Dict, List, Optional, Tuple

from typing import Dict

from common.orm.fields.PrimaryKeyField import PrimaryKeyField

from common.orm.fields.StringField import StringField

from common.orm.model.Model import Model



LOGGER = logging.getLogger(__name__)





class PolicyRuleModel(Model):

    pk = PrimaryKeyField()

    value = StringField(required=True, allow_empty=False)



    def dump_id(self) -> Dict:

        return {

            "uuid": {"uuid": self.pk}

        }

        return {'uuid': {'uuid': self.pk}}



    def dump(self) -> Dict:

        return json.loads(self.value)

src/context/service/grpc_server/Constants.py

+1 −3
Original line number Diff line number Diff line
@@ -19,9 +19,7 @@ TOPIC_DEVICE = 'device' 
TOPIC_LINK       = 'link'

TOPIC_SERVICE    = 'service'

TOPIC_SLICE      = 'slice'

TOPIC_POLICY     = 'policy'



TOPICS = {TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_TOPOLOGY, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE,

          TOPIC_POLICY}

TOPICS = {TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_TOPOLOGY, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE}



CONSUME_TIMEOUT = 0.5 # seconds

src/context/service/grpc_server/ContextServiceServicerImpl.py

+13 −15
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ from common.proto.context_pb2_grpc import ContextServiceServicer 
from common.proto.context_policy_pb2_grpc import ContextPolicyServiceServicer

from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method

from common.rpc_method_wrapper.ServiceExceptions import InvalidArgumentException

from common.tools.grpc.Tools import grpc_message_to_json

from context.service.database.ConfigModel import update_config

from context.service.database.ConnectionModel import ConnectionModel, set_path

from context.service.database.ConstraintModel import set_constraints
@@ -51,8 +52,7 @@ from context.service.database.SliceModel import SliceModel, grpc_to_enum__slice_ 
from context.service.database.TopologyModel import TopologyModel

from .Constants import (

    CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_SERVICE, TOPIC_SLICE,

    TOPIC_TOPOLOGY, TOPIC_POLICY)

from common.tools.grpc.Tools import grpc_message_to_json_string, grpc_message_to_json

    TOPIC_TOPOLOGY)



LOGGER = logging.getLogger(__name__)
@@ -820,7 +820,8 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
            yield ConnectionEvent(**json.loads(message.content))





    # ----- Policy ----------------------------------------------------------------------------------------------------

    # ----- Policy -----------------------------------------------------------------------------------------------------



    @safe_and_metered_rpc_method(METRICS, LOGGER)

    def ListPolicyRuleIds(self, request: Empty, context: grpc.ServicerContext) -> PolicyRuleIdList:

        with self.lock:
@@ -833,8 +834,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
        with self.lock:

            db_policy_rules: List[PolicyRuleModel] = get_all_objects(self.database, PolicyRuleModel)

            db_policy_rules = sorted(db_policy_rules, key=operator.attrgetter('pk'))

            res = PolicyRuleList(policyRules=[db_policy_rule.dump_id() for db_policy_rule in db_policy_rules])

            return res

            return PolicyRuleList(policyRules=[db_policy_rule.dump() for db_policy_rule in db_policy_rules])



    @safe_and_metered_rpc_method(METRICS, LOGGER)

    def GetPolicyRule(self, request: PolicyRuleId, context: grpc.ServicerContext) -> PolicyRule:
@@ -848,15 +848,14 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
        with self.lock:

            policy_rule_type = request.WhichOneof('policy_rule')

            policy_rule_json = grpc_message_to_json(request)

            policy_rule_uuid = policy_rule_json[policy_rule_type]["policyRuleBasic"]["policyRuleId"]["uuid"]["uuid"]

            policy_rule_uuid = policy_rule_json[policy_rule_type]['policyRuleBasic']['policyRuleId']['uuid']['uuid']

            result: Tuple[PolicyRuleModel, bool] = update_or_create_object(

                self.database, PolicyRuleModel, policy_rule_uuid, {"value": json.dumps(policy_rule_json)})

            db_policy, updated = result

                self.database, PolicyRuleModel, policy_rule_uuid, {'value': json.dumps(policy_rule_json)})

            db_policy, updated = result # pylint: disable=unused-variable



            event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

            #event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

            dict_policy_id = db_policy.dump_id()

            notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})



            #notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            return PolicyRuleId(**dict_policy_id)



    @safe_and_metered_rpc_method(METRICS, LOGGER)
@@ -865,11 +864,10 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 
            policy_uuid = request.uuid.uuid

            db_policy = PolicyRuleModel(self.database, policy_uuid, auto_load=False)

            found = db_policy.load()

            if not found:

                return Empty()

            if not found: return Empty()



            dict_policy_id = db_policy.dump_id()

            db_policy.delete()

            event_type = EventTypeEnum.EVENTTYPE_REMOVE

            notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            #event_type = EventTypeEnum.EVENTTYPE_REMOVE

            #notify_event(self.messagebroker, TOPIC_POLICY, event_type, {"policy_id": dict_policy_id})

            return Empty()

src/context/service/rest_server/Resources.py

+22 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ from flask.json import jsonify 
from flask_restful import Resource

from common.orm.Database import Database

from common.proto.context_pb2 import ConnectionId, ContextId, DeviceId, Empty, LinkId, ServiceId, SliceId, TopologyId

from common.proto.policy_pb2 import PolicyRuleId

from common.tools.grpc.Tools import grpc_message_to_json

from context.service.grpc_server.ContextServiceServicerImpl import ContextServiceServicerImpl
@@ -61,6 +62,11 @@ def grpc_topology_id(context_uuid, topology_uuid): 
        'topology_uuid': {'uuid': topology_uuid}

    })



def grpc_policy_rule_id(policy_rule_uuid):

    return PolicyRuleId(**{

        'uuid': {'uuid': policy_rule_uuid}

    })



class _Resource(Resource):

    def __init__(self, database : Database) -> None:

        super().__init__()
@@ -151,6 +157,18 @@ class Connection(_Resource): 
    def get(self, connection_uuid : str):

        return format_grpc_to_json(self.servicer.GetConnection(grpc_connection_id(connection_uuid), None))



class PolicyRuleIds(_Resource):

    def get(self):

        return format_grpc_to_json(self.servicer.ListPolicyRuleIds(Empty(), None))



class PolicyRules(_Resource):

    def get(self):

        return format_grpc_to_json(self.servicer.ListPolicyRules(Empty(), None))



class PolicyRule(_Resource):

    def get(self, policy_rule_uuid : str):

        return format_grpc_to_json(self.servicer.GetPolicyRule(grpc_policy_rule_id(policy_rule_uuid), None))



class DumpText(Resource):

    def __init__(self, database : Database) -> None:

        super().__init__()
@@ -219,6 +237,10 @@ RESOURCES = [ 
    ('api.connections',    Connections,   '/context/<string:context_uuid>/service/<path:service_uuid>/connections'),

    ('api.connection',     Connection,    '/connection/<path:connection_uuid>'),



    ('api.policyrule_ids', PolicyRuleIds, '/policyrule_ids'),

    ('api.policyrules',    PolicyRules,   '/policyrules'),

    ('api.policyrule',     PolicyRule,    '/policyrule/<string:policyrule_uuid>'),



    ('api.dump.text',      DumpText,      '/dump/text'),

    ('api.dump.html',      DumpHtml,      '/dump/html'),

]