diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index f5b2662b37865f1bdb39d4a053dc3b297fd56e82..82e28a7f13135735909a99b635cf26bb3e02e252 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -39,8 +39,8 @@ from .database.Service import service_delete, service_get, service_list_ids, ser from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_set, slice_unset from .database.Topology import topology_delete, topology_get, topology_list_ids, topology_list_objs, topology_set from .Events import ( - CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY, - TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY, notify_event) + CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_POLICY, TOPIC_SERVICE, + TOPIC_SLICE, TOPIC_TOPOLOGY, notify_event) LOGGER = logging.getLogger(__name__) @@ -313,22 +313,27 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ListPolicyRuleIds(self, request : Empty, context: grpc.ServicerContext) -> PolicyRuleIdList: - return policyrule_list_ids(self.db_engine) + return PolicyRuleIdList(policyRuleIdList=policyrule_list_ids(self.db_engine)) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ListPolicyRules(self, request : Empty, context: grpc.ServicerContext) -> PolicyRuleList: - return policyrule_list_objs(self.db_engine) + return PolicyRuleList(policyRules=policyrule_list_objs(self.db_engine)) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetPolicyRule(self, request : PolicyRuleId, context: grpc.ServicerContext) -> PolicyRule: - return policyrule_get(self.db_engine, request) + return PolicyRule(**policyrule_get(self.db_engine, request)) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SetPolicyRule(self, request : PolicyRule, context: grpc.ServicerContext) -> PolicyRuleId: - policyrule_id,updated = policyrule_set(self.db_engine, request) # pylint: disable=unused-variable - return policyrule_id + policyrule_id,updated = policyrule_set(self.db_engine, request) + event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE + notify_event(self.messagebroker, TOPIC_POLICY, event_type, {'policyrule_id': policyrule_id}) + return PolicyRuleId(**policyrule_id) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def RemovePolicyRule(self, request : PolicyRuleId, context: grpc.ServicerContext) -> Empty: - deleted = policyrule_delete(self.db_engine, request) # pylint: disable=unused-variable + policyrule_id,deleted = policyrule_delete(self.db_engine, request) + if deleted: + event_type = EventTypeEnum.EVENTTYPE_REMOVE + notify_event(self.messagebroker, TOPIC_POLICY, event_type, {'policyrule_id': policyrule_id}) return Empty() diff --git a/src/context/service/Events.py b/src/context/service/Events.py index e7cf1997c349652fae405bb191d4fe03b4b0a238..77401314bfc7f6682a2d1515cbfce26ebf123332 100644 --- a/src/context/service/Events.py +++ b/src/context/service/Events.py @@ -22,14 +22,13 @@ TOPIC_CONNECTION = 'connection' TOPIC_CONTEXT = 'context' TOPIC_DEVICE = 'device' TOPIC_LINK = 'link' -#TOPIC_POLICY = 'policy' +TOPIC_POLICY = 'policy' TOPIC_SERVICE = 'service' TOPIC_SLICE = 'slice' TOPIC_TOPOLOGY = 'topology' TOPICS = { - TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY, - TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY + TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_POLICY, TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY } CONSUME_TIMEOUT = 0.5 # seconds diff --git a/src/context/service/database/Connection.py b/src/context/service/database/Connection.py index f1616e96e715b7f63722d8e939b7851c029d04a0..6d6d941cbe9cbe21c776a65398cd3ad3bb2073cd 100644 --- a/src/context/service/database/Connection.py +++ b/src/context/service/database/Connection.py @@ -136,7 +136,7 @@ def connection_set(db_engine : Engine, request : Connection) -> Tuple[Dict, bool return updated updated = run_transaction(sessionmaker(bind=db_engine), callback) - return ConnectionId(**json_connection_id(connection_uuid)),updated + return json_connection_id(connection_uuid),updated def connection_delete(db_engine : Engine, request : ConnectionId) -> Tuple[Dict, bool]: connection_uuid = connection_get_uuid(request, allow_random=False) @@ -144,4 +144,4 @@ def connection_delete(db_engine : Engine, request : ConnectionId) -> Tuple[Dict, num_deleted = session.query(ConnectionModel).filter_by(connection_uuid=connection_uuid).delete() return num_deleted > 0 deleted = run_transaction(sessionmaker(bind=db_engine), callback) - return ConnectionId(**json_connection_id(connection_uuid)),deleted + return json_connection_id(connection_uuid),deleted diff --git a/src/context/service/database/PolicyRule.py b/src/context/service/database/PolicyRule.py index 2371af88e12e6eb1fef65ee26f681bcf03f6f71c..70a37c7d8aaf9e412e330fc6060142ea591eaee0 100644 --- a/src/context/service/database/PolicyRule.py +++ b/src/context/service/database/PolicyRule.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +import datetime, json from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker @@ -28,19 +28,19 @@ from .models.PolicyRuleModel import PolicyRuleDeviceModel, PolicyRuleKindEnum, P from .uuids.PolicuRule import policyrule_get_uuid from .uuids.Service import service_get_uuid -def policyrule_list_ids(db_engine : Engine) -> PolicyRuleIdList: +def policyrule_list_ids(db_engine : Engine) -> List[Dict]: def callback(session : Session) -> List[Dict]: obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all() #.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none() return [obj.dump_id() for obj in obj_list] - return PolicyRuleIdList(policyRuleIdList=run_transaction(sessionmaker(bind=db_engine), callback)) + return run_transaction(sessionmaker(bind=db_engine), callback) -def policyrule_list_objs(db_engine : Engine) -> PolicyRuleList: +def policyrule_list_objs(db_engine : Engine) -> List[Dict]: def callback(session : Session) -> List[Dict]: obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all() #.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none() return [obj.dump() for obj in obj_list] - return PolicyRuleList(policyRules=run_transaction(sessionmaker(bind=db_engine), callback)) + return run_transaction(sessionmaker(bind=db_engine), callback) def policyrule_get(db_engine : Engine, request : PolicyRuleId) -> PolicyRule: policyrule_uuid = policyrule_get_uuid(request, allow_random=False) @@ -54,7 +54,7 @@ def policyrule_get(db_engine : Engine, request : PolicyRuleId) -> PolicyRule: raise NotFoundException('PolicyRule', raw_policyrule_uuid, extra_details=[ 'policyrule_uuid generated was: {:s}'.format(policyrule_uuid) ]) - return PolicyRule(**obj) + return obj def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRuleId, bool]: policyrule_kind = request.WhichOneof('policy_rule') @@ -74,6 +74,8 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule 'actionList': json_policyrule_basic.get('actionList', []), }, sort_keys=True) + now = datetime.datetime.utcnow() + policyrule_data = [{ 'policyrule_uuid' : policyrule_uuid, 'policyrule_kind' : policyrule_kind, @@ -81,6 +83,8 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule 'policyrule_state_message': policyrule_state_message, 'policyrule_priority' : policyrule_basic.priority, 'policyrule_eca_data' : policyrule_eca_data, + 'created_at' : now, + 'updated_at' : now, }] policyrule_service_uuid = None @@ -99,7 +103,7 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule }) device_uuids.add(device_uuid) - def callback(session : Session) -> None: + def callback(session : Session) -> bool: stmt = insert(PolicyRuleModel).values(policyrule_data) stmt = stmt.on_conflict_do_update( index_elements=[PolicyRuleModel.policyrule_uuid], @@ -108,22 +112,27 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule policyrule_state_message = stmt.excluded.policyrule_state_message, policyrule_priority = stmt.excluded.policyrule_priority, policyrule_eca_data = stmt.excluded.policyrule_eca_data, + updated_at = stmt.excluded.updated_at, ) ) - session.execute(stmt) + stmt = stmt.returning(PolicyRuleModel.created_at, PolicyRuleModel.updated_at) + created_at,updated_at = session.execute(stmt).fetchone() + updated = updated_at > created_at if len(related_devices) > 0: session.execute(insert(PolicyRuleDeviceModel).values(related_devices).on_conflict_do_nothing( index_elements=[PolicyRuleDeviceModel.policyrule_uuid, PolicyRuleDeviceModel.device_uuid] )) - run_transaction(sessionmaker(bind=db_engine), callback) - updated = False # TODO: improve and check if created/updated - return PolicyRuleId(**json_policyrule_id(policyrule_uuid)),updated + return updated + + updated = run_transaction(sessionmaker(bind=db_engine), callback) + return json_policyrule_id(policyrule_uuid),updated -def policyrule_delete(db_engine : Engine, request : PolicyRuleId) -> bool: +def policyrule_delete(db_engine : Engine, request : PolicyRuleId) -> Tuple[Dict, bool]: policyrule_uuid = policyrule_get_uuid(request, allow_random=False) def callback(session : Session) -> bool: num_deleted = session.query(PolicyRuleModel).filter_by(policyrule_uuid=policyrule_uuid).delete() return num_deleted > 0 - return run_transaction(sessionmaker(bind=db_engine), callback) + deleted = run_transaction(sessionmaker(bind=db_engine), callback) + return json_policyrule_id(policyrule_uuid),deleted diff --git a/src/context/service/database/Service.py b/src/context/service/database/Service.py index a8f9f40d6d6351450e7fe1c6e400a0b36f9b3e8b..b65010fed00c885833ecab11fa0bdc5e2a56a4fa 100644 --- a/src/context/service/database/Service.py +++ b/src/context/service/database/Service.py @@ -111,6 +111,7 @@ def service_set(db_engine : Engine, request : Service) -> Tuple[Dict, bool]: service_name = stmt.excluded.service_name, service_type = stmt.excluded.service_type, service_status = stmt.excluded.service_status, + updated_at = stmt.excluded.updated_at, ) ) stmt = stmt.returning(ServiceModel.created_at, ServiceModel.updated_at) diff --git a/src/context/service/database/Slice.py b/src/context/service/database/Slice.py index f255968b28902d02182e29d4e16d4efb1302b632..b0b83238c8bcfbac826cc13789413e2ae957f47c 100644 --- a/src/context/service/database/Slice.py +++ b/src/context/service/database/Slice.py @@ -127,6 +127,7 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]: set_=dict( slice_name = stmt.excluded.slice_name, slice_status = stmt.excluded.slice_status, + updated_at = stmt.excluded.updated_at, slice_owner_uuid = stmt.excluded.slice_owner_uuid, slice_owner_string = stmt.excluded.slice_owner_string, ) diff --git a/src/context/service/database/models/PolicyRuleModel.py b/src/context/service/database/models/PolicyRuleModel.py index 8fc1110870a65af78369e45540b0e7bfb9144c41..4ccec8dd8fe5c3e6c0a00fea691c9e380c8fddcd 100644 --- a/src/context/service/database/models/PolicyRuleModel.py +++ b/src/context/service/database/models/PolicyRuleModel.py @@ -13,12 +13,11 @@ # limitations under the License. import enum, json -from sqlalchemy import CheckConstraint, Column, Enum, ForeignKey, Integer, String +from sqlalchemy import CheckConstraint, Column, DateTime, Enum, ForeignKey, Integer, String from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from typing import Dict - -from context.service.database.models.enums.PolicyRuleState import ORM_PolicyRuleStateEnum +from .enums.PolicyRuleState import ORM_PolicyRuleStateEnum from ._Base import _Base # Enum values should match name of field in PolicyRule message @@ -36,6 +35,8 @@ class PolicyRuleModel(_Base): policyrule_priority = Column(Integer, nullable=False) policyrule_service_uuid = Column(ForeignKey('service.service_uuid', ondelete='RESTRICT'), nullable=True) policyrule_eca_data = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) policyrule_service = relationship('ServiceModel') # back_populates='policyrules' policyrule_devices = relationship('PolicyRuleDeviceModel' ) # back_populates='policyrule' diff --git a/src/context/tests/Constants.py b/src/context/tests/Constants.py new file mode 100644 index 0000000000000000000000000000000000000000..b29584a7b743db8c4cc75dbe9418b42142797fed --- /dev/null +++ b/src/context/tests/Constants.py @@ -0,0 +1,15 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GET_EVENTS_TIMEOUT = 60.0 diff --git a/src/context/tests/test_connection.py b/src/context/tests/test_connection.py index 909ddb6efe2c4613b40050162fc9a4ba27dce304..86abad7ed6a138128094e544adb6936e976ee477 100644 --- a/src/context/tests/test_connection.py +++ b/src/context/tests/test_connection.py @@ -19,13 +19,12 @@ from context.client.ContextClient import ContextClient from context.service.database.uuids.Connection import connection_get_uuid from context.service.database.uuids.EndPoint import endpoint_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import ( CONNECTION_R1_R3, CONNECTION_R1_R3_ID, CONNECTION_R1_R3_NAME, CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID, DEVICE_R2, DEVICE_R2_ID, DEVICE_R3, DEVICE_R3_ID, SERVICE_R1_R2, SERVICE_R1_R2_ID, SERVICE_R1_R3, SERVICE_R1_R3_ID, SERVICE_R2_R3, SERVICE_R2_R3_ID, TOPOLOGY, TOPOLOGY_ID) -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_service.py::test_service', 'context/tests/test_slice.py::test_slice']) def test_connection(context_client : ContextClient) -> None: diff --git a/src/context/tests/test_context.py b/src/context/tests/test_context.py index 77f1dc3805a4ad585b54a191a93390f62bc40425..7a9564df649da3d736589b462cbd6013d4d04d30 100644 --- a/src/context/tests/test_context.py +++ b/src/context/tests/test_context.py @@ -17,10 +17,9 @@ from common.proto.context_pb2 import Context, ContextEvent, ContextId, Empty, Ev from context.client.ContextClient import ContextClient from context.service.database.uuids.Context import context_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import CONTEXT, CONTEXT_ID, CONTEXT_NAME -GET_EVENTS_TIMEOUT = 10.0 - def test_context(context_client : ContextClient) -> None: # ----- Initialize the EventsCollector ----------------------------------------------------------------------------- diff --git a/src/context/tests/test_device.py b/src/context/tests/test_device.py index bcbe4cc3b4319609fbb364dff1171c39e8be4989..615ebe0beb503b891e4384e690adb76e028da92a 100644 --- a/src/context/tests/test_device.py +++ b/src/context/tests/test_device.py @@ -19,10 +19,9 @@ from common.proto.context_pb2 import ( from context.client.ContextClient import ContextClient from context.service.database.uuids.Device import device_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID, DEVICE_R1_NAME, TOPOLOGY, TOPOLOGY_ID -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_topology.py::test_topology']) def test_device(context_client : ContextClient) -> None: diff --git a/src/context/tests/test_link.py b/src/context/tests/test_link.py index c8ed1d486ba37ead3c79bb8c25506669c656e764..e56a1889d3267b776953c053b36b8645a5f7cad7 100644 --- a/src/context/tests/test_link.py +++ b/src/context/tests/test_link.py @@ -19,12 +19,11 @@ from common.proto.context_pb2 import ( from context.client.ContextClient import ContextClient from context.client.EventsCollector import EventsCollector from context.service.database.uuids.Link import link_get_uuid +from .Constants import GET_EVENTS_TIMEOUT from .Objects import ( CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID, DEVICE_R2, DEVICE_R2_ID, LINK_R1_R2, LINK_R1_R2_ID, LINK_R1_R2_NAME, TOPOLOGY, TOPOLOGY_ID) -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_device.py::test_device']) def test_link(context_client : ContextClient) -> None: diff --git a/src/context/tests/test_policy.py b/src/context/tests/test_policy.py index f9bf5ef6d624a5e47e5fbd39dc389aa53757946f..1cc0b955760329c08a50e25f3240b93be76a2fcc 100644 --- a/src/context/tests/test_policy.py +++ b/src/context/tests/test_policy.py @@ -19,7 +19,7 @@ from context.client.ContextClient import ContextClient from context.service.database.uuids.PolicuRule import policyrule_get_uuid from .Objects import POLICYRULE, POLICYRULE_ID, POLICYRULE_NAME -@pytest.mark.depends(on=['context/tests/test_device.py::test_device', 'context/tests/test_service.py::test_service']) +@pytest.mark.depends(on=['context/tests/test_connection.py::test_connection']) def test_policy(context_client : ContextClient): # ----- Get when the object does not exist ------------------------------------------------------------------------- diff --git a/src/context/tests/test_service.py b/src/context/tests/test_service.py index 4e46c24ad90329aac580b9ea92e21a33dd0263be..ca02a4a9193b75f67b5ef9ac50ecaa47fa27f1c6 100644 --- a/src/context/tests/test_service.py +++ b/src/context/tests/test_service.py @@ -19,12 +19,11 @@ from common.proto.context_pb2 import ( from context.client.ContextClient import ContextClient from context.service.database.uuids.Service import service_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import ( CONTEXT, CONTEXT_ID, CONTEXT_NAME, DEVICE_R1, DEVICE_R1_ID, SERVICE_R1_R2_NAME, DEVICE_R2, DEVICE_R2_ID, SERVICE_R1_R2, SERVICE_R1_R2_ID, TOPOLOGY, TOPOLOGY_ID) -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_link.py::test_link']) def test_service(context_client : ContextClient) -> None: diff --git a/src/context/tests/test_slice.py b/src/context/tests/test_slice.py index 6996bb39e74aeccdaec261850925cff217dcb842..1008e7e91824d11504876971dd8d08ad707c5039 100644 --- a/src/context/tests/test_slice.py +++ b/src/context/tests/test_slice.py @@ -19,13 +19,12 @@ from common.proto.context_pb2 import ( from context.client.ContextClient import ContextClient from context.service.database.uuids.Slice import slice_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import ( CONTEXT, CONTEXT_ID, CONTEXT_NAME, DEVICE_R1, DEVICE_R1_ID, DEVICE_R2, DEVICE_R2_ID, DEVICE_R3, DEVICE_R3_ID, LINK_R1_R2, LINK_R1_R2_ID, LINK_R1_R3, LINK_R1_R3_ID, LINK_R2_R3, LINK_R2_R3_ID, SERVICE_R1_R2, SERVICE_R1_R2_ID, SERVICE_R2_R3, SERVICE_R2_R3_ID, SLICE_R1_R3, SLICE_R1_R3_ID, SLICE_R1_R3_NAME, TOPOLOGY, TOPOLOGY_ID) -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_service.py::test_service']) def test_slice(context_client : ContextClient) -> None: diff --git a/src/context/tests/test_topology.py b/src/context/tests/test_topology.py index 6a3367d49ae004a4c9670d04256f2f726f125a52..0d8b8c0272464ac1b996189b3fd1b60b9341f9df 100644 --- a/src/context/tests/test_topology.py +++ b/src/context/tests/test_topology.py @@ -18,10 +18,9 @@ from common.proto.context_pb2 import ( from context.client.ContextClient import ContextClient from context.service.database.uuids.Topology import topology_get_uuid from context.client.EventsCollector import EventsCollector +from .Constants import GET_EVENTS_TIMEOUT from .Objects import CONTEXT, CONTEXT_ID, CONTEXT_NAME, TOPOLOGY, TOPOLOGY_ID, TOPOLOGY_NAME -GET_EVENTS_TIMEOUT = 10.0 - @pytest.mark.depends(on=['context/tests/test_context.py::test_context']) def test_topology(context_client : ContextClient) -> None: