From 362b28a9d147a4cb960efea553dfb8ba9d06b633 Mon Sep 17 00:00:00 2001
From: gifrerenom <lluis.gifre@cttc.es>
Date: Mon, 16 Jan 2023 12:03:32 +0000
Subject: [PATCH] Context:
- corrected unitary test order and one-by-one execution
- extractedunitary test constant to separate file
- added updated_at refresh for Service and Slice entities
- corrected return types for Connection entity
- prepared PolicyRule entity to raise events and corrected return types of methods
---
.../service/ContextServiceServicerImpl.py | 21 ++++++-----
src/context/service/Events.py | 5 ++-
src/context/service/database/Connection.py | 4 +--
src/context/service/database/PolicyRule.py | 35 ++++++++++++-------
src/context/service/database/Service.py | 1 +
src/context/service/database/Slice.py | 1 +
.../database/models/PolicyRuleModel.py | 7 ++--
src/context/tests/Constants.py | 15 ++++++++
src/context/tests/test_connection.py | 3 +-
src/context/tests/test_context.py | 3 +-
src/context/tests/test_device.py | 3 +-
src/context/tests/test_link.py | 3 +-
src/context/tests/test_policy.py | 2 +-
src/context/tests/test_service.py | 3 +-
src/context/tests/test_slice.py | 3 +-
src/context/tests/test_topology.py | 3 +-
16 files changed, 68 insertions(+), 44 deletions(-)
create mode 100644 src/context/tests/Constants.py
diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py
index f5b2662b3..82e28a7f1 100644
--- a/src/context/service/ContextServiceServicerImpl.py
+++ b/src/context/service/ContextServiceServicerImpl.py
@@ -39,8 +39,8 @@ from .database.Service import service_delete, service_get, service_list_ids, ser
from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_set, slice_unset
from .database.Topology import topology_delete, topology_get, topology_list_ids, topology_list_objs, topology_set
from .Events import (
- CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY,
- TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY, notify_event)
+ CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_POLICY, TOPIC_SERVICE,
+ TOPIC_SLICE, TOPIC_TOPOLOGY, notify_event)
LOGGER = logging.getLogger(__name__)
@@ -313,22 +313,27 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def ListPolicyRuleIds(self, request : Empty, context: grpc.ServicerContext) -> PolicyRuleIdList:
- return policyrule_list_ids(self.db_engine)
+ return PolicyRuleIdList(policyRuleIdList=policyrule_list_ids(self.db_engine))
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def ListPolicyRules(self, request : Empty, context: grpc.ServicerContext) -> PolicyRuleList:
- return policyrule_list_objs(self.db_engine)
+ return PolicyRuleList(policyRules=policyrule_list_objs(self.db_engine))
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def GetPolicyRule(self, request : PolicyRuleId, context: grpc.ServicerContext) -> PolicyRule:
- return policyrule_get(self.db_engine, request)
+ return PolicyRule(**policyrule_get(self.db_engine, request))
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def SetPolicyRule(self, request : PolicyRule, context: grpc.ServicerContext) -> PolicyRuleId:
- policyrule_id,updated = policyrule_set(self.db_engine, request) # pylint: disable=unused-variable
- return policyrule_id
+ policyrule_id,updated = policyrule_set(self.db_engine, request)
+ event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE
+ notify_event(self.messagebroker, TOPIC_POLICY, event_type, {'policyrule_id': policyrule_id})
+ return PolicyRuleId(**policyrule_id)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def RemovePolicyRule(self, request : PolicyRuleId, context: grpc.ServicerContext) -> Empty:
- deleted = policyrule_delete(self.db_engine, request) # pylint: disable=unused-variable
+ policyrule_id,deleted = policyrule_delete(self.db_engine, request)
+ if deleted:
+ event_type = EventTypeEnum.EVENTTYPE_REMOVE
+ notify_event(self.messagebroker, TOPIC_POLICY, event_type, {'policyrule_id': policyrule_id})
return Empty()
diff --git a/src/context/service/Events.py b/src/context/service/Events.py
index e7cf1997c..77401314b 100644
--- a/src/context/service/Events.py
+++ b/src/context/service/Events.py
@@ -22,14 +22,13 @@ TOPIC_CONNECTION = 'connection'
TOPIC_CONTEXT = 'context'
TOPIC_DEVICE = 'device'
TOPIC_LINK = 'link'
-#TOPIC_POLICY = 'policy'
+TOPIC_POLICY = 'policy'
TOPIC_SERVICE = 'service'
TOPIC_SLICE = 'slice'
TOPIC_TOPOLOGY = 'topology'
TOPICS = {
- TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY,
- TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY
+ TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, TOPIC_POLICY, TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY
}
CONSUME_TIMEOUT = 0.5 # seconds
diff --git a/src/context/service/database/Connection.py b/src/context/service/database/Connection.py
index f1616e96e..6d6d941cb 100644
--- a/src/context/service/database/Connection.py
+++ b/src/context/service/database/Connection.py
@@ -136,7 +136,7 @@ def connection_set(db_engine : Engine, request : Connection) -> Tuple[Dict, bool
return updated
updated = run_transaction(sessionmaker(bind=db_engine), callback)
- return ConnectionId(**json_connection_id(connection_uuid)),updated
+ return json_connection_id(connection_uuid),updated
def connection_delete(db_engine : Engine, request : ConnectionId) -> Tuple[Dict, bool]:
connection_uuid = connection_get_uuid(request, allow_random=False)
@@ -144,4 +144,4 @@ def connection_delete(db_engine : Engine, request : ConnectionId) -> Tuple[Dict,
num_deleted = session.query(ConnectionModel).filter_by(connection_uuid=connection_uuid).delete()
return num_deleted > 0
deleted = run_transaction(sessionmaker(bind=db_engine), callback)
- return ConnectionId(**json_connection_id(connection_uuid)),deleted
+ return json_connection_id(connection_uuid),deleted
diff --git a/src/context/service/database/PolicyRule.py b/src/context/service/database/PolicyRule.py
index 2371af88e..70a37c7d8 100644
--- a/src/context/service/database/PolicyRule.py
+++ b/src/context/service/database/PolicyRule.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import json
+import datetime, json
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
@@ -28,19 +28,19 @@ from .models.PolicyRuleModel import PolicyRuleDeviceModel, PolicyRuleKindEnum, P
from .uuids.PolicuRule import policyrule_get_uuid
from .uuids.Service import service_get_uuid
-def policyrule_list_ids(db_engine : Engine) -> PolicyRuleIdList:
+def policyrule_list_ids(db_engine : Engine) -> List[Dict]:
def callback(session : Session) -> List[Dict]:
obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all()
#.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none()
return [obj.dump_id() for obj in obj_list]
- return PolicyRuleIdList(policyRuleIdList=run_transaction(sessionmaker(bind=db_engine), callback))
+ return run_transaction(sessionmaker(bind=db_engine), callback)
-def policyrule_list_objs(db_engine : Engine) -> PolicyRuleList:
+def policyrule_list_objs(db_engine : Engine) -> List[Dict]:
def callback(session : Session) -> List[Dict]:
obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all()
#.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none()
return [obj.dump() for obj in obj_list]
- return PolicyRuleList(policyRules=run_transaction(sessionmaker(bind=db_engine), callback))
+ return run_transaction(sessionmaker(bind=db_engine), callback)
def policyrule_get(db_engine : Engine, request : PolicyRuleId) -> PolicyRule:
policyrule_uuid = policyrule_get_uuid(request, allow_random=False)
@@ -54,7 +54,7 @@ def policyrule_get(db_engine : Engine, request : PolicyRuleId) -> PolicyRule:
raise NotFoundException('PolicyRule', raw_policyrule_uuid, extra_details=[
'policyrule_uuid generated was: {:s}'.format(policyrule_uuid)
])
- return PolicyRule(**obj)
+ return obj
def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRuleId, bool]:
policyrule_kind = request.WhichOneof('policy_rule')
@@ -74,6 +74,8 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule
'actionList': json_policyrule_basic.get('actionList', []),
}, sort_keys=True)
+ now = datetime.datetime.utcnow()
+
policyrule_data = [{
'policyrule_uuid' : policyrule_uuid,
'policyrule_kind' : policyrule_kind,
@@ -81,6 +83,8 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule
'policyrule_state_message': policyrule_state_message,
'policyrule_priority' : policyrule_basic.priority,
'policyrule_eca_data' : policyrule_eca_data,
+ 'created_at' : now,
+ 'updated_at' : now,
}]
policyrule_service_uuid = None
@@ -99,7 +103,7 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule
})
device_uuids.add(device_uuid)
- def callback(session : Session) -> None:
+ def callback(session : Session) -> bool:
stmt = insert(PolicyRuleModel).values(policyrule_data)
stmt = stmt.on_conflict_do_update(
index_elements=[PolicyRuleModel.policyrule_uuid],
@@ -108,22 +112,27 @@ def policyrule_set(db_engine : Engine, request : PolicyRule) -> Tuple[PolicyRule
policyrule_state_message = stmt.excluded.policyrule_state_message,
policyrule_priority = stmt.excluded.policyrule_priority,
policyrule_eca_data = stmt.excluded.policyrule_eca_data,
+ updated_at = stmt.excluded.updated_at,
)
)
- session.execute(stmt)
+ stmt = stmt.returning(PolicyRuleModel.created_at, PolicyRuleModel.updated_at)
+ created_at,updated_at = session.execute(stmt).fetchone()
+ updated = updated_at > created_at
if len(related_devices) > 0:
session.execute(insert(PolicyRuleDeviceModel).values(related_devices).on_conflict_do_nothing(
index_elements=[PolicyRuleDeviceModel.policyrule_uuid, PolicyRuleDeviceModel.device_uuid]
))
- run_transaction(sessionmaker(bind=db_engine), callback)
- updated = False # TODO: improve and check if created/updated
- return PolicyRuleId(**json_policyrule_id(policyrule_uuid)),updated
+ return updated
+
+ updated = run_transaction(sessionmaker(bind=db_engine), callback)
+ return json_policyrule_id(policyrule_uuid),updated
-def policyrule_delete(db_engine : Engine, request : PolicyRuleId) -> bool:
+def policyrule_delete(db_engine : Engine, request : PolicyRuleId) -> Tuple[Dict, bool]:
policyrule_uuid = policyrule_get_uuid(request, allow_random=False)
def callback(session : Session) -> bool:
num_deleted = session.query(PolicyRuleModel).filter_by(policyrule_uuid=policyrule_uuid).delete()
return num_deleted > 0
- return run_transaction(sessionmaker(bind=db_engine), callback)
+ deleted = run_transaction(sessionmaker(bind=db_engine), callback)
+ return json_policyrule_id(policyrule_uuid),deleted
diff --git a/src/context/service/database/Service.py b/src/context/service/database/Service.py
index a8f9f40d6..b65010fed 100644
--- a/src/context/service/database/Service.py
+++ b/src/context/service/database/Service.py
@@ -111,6 +111,7 @@ def service_set(db_engine : Engine, request : Service) -> Tuple[Dict, bool]:
service_name = stmt.excluded.service_name,
service_type = stmt.excluded.service_type,
service_status = stmt.excluded.service_status,
+ updated_at = stmt.excluded.updated_at,
)
)
stmt = stmt.returning(ServiceModel.created_at, ServiceModel.updated_at)
diff --git a/src/context/service/database/Slice.py b/src/context/service/database/Slice.py
index f255968b2..b0b83238c 100644
--- a/src/context/service/database/Slice.py
+++ b/src/context/service/database/Slice.py
@@ -127,6 +127,7 @@ def slice_set(db_engine : Engine, request : Slice) -> Tuple[Dict, bool]:
set_=dict(
slice_name = stmt.excluded.slice_name,
slice_status = stmt.excluded.slice_status,
+ updated_at = stmt.excluded.updated_at,
slice_owner_uuid = stmt.excluded.slice_owner_uuid,
slice_owner_string = stmt.excluded.slice_owner_string,
)
diff --git a/src/context/service/database/models/PolicyRuleModel.py b/src/context/service/database/models/PolicyRuleModel.py
index 8fc111087..4ccec8dd8 100644
--- a/src/context/service/database/models/PolicyRuleModel.py
+++ b/src/context/service/database/models/PolicyRuleModel.py
@@ -13,12 +13,11 @@
# limitations under the License.
import enum, json
-from sqlalchemy import CheckConstraint, Column, Enum, ForeignKey, Integer, String
+from sqlalchemy import CheckConstraint, Column, DateTime, Enum, ForeignKey, Integer, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import relationship
from typing import Dict
-
-from context.service.database.models.enums.PolicyRuleState import ORM_PolicyRuleStateEnum
+from .enums.PolicyRuleState import ORM_PolicyRuleStateEnum
from ._Base import _Base
# Enum values should match name of field in PolicyRule message
@@ -36,6 +35,8 @@ class PolicyRuleModel(_Base):
policyrule_priority = Column(Integer, nullable=False)
policyrule_service_uuid = Column(ForeignKey('service.service_uuid', ondelete='RESTRICT'), nullable=True)
policyrule_eca_data = Column(String, nullable=False)
+ created_at = Column(DateTime, nullable=False)
+ updated_at = Column(DateTime, nullable=False)
policyrule_service = relationship('ServiceModel') # back_populates='policyrules'
policyrule_devices = relationship('PolicyRuleDeviceModel' ) # back_populates='policyrule'
diff --git a/src/context/tests/Constants.py b/src/context/tests/Constants.py
new file mode 100644
index 000000000..b29584a7b
--- /dev/null
+++ b/src/context/tests/Constants.py
@@ -0,0 +1,15 @@
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GET_EVENTS_TIMEOUT = 60.0
diff --git a/src/context/tests/test_connection.py b/src/context/tests/test_connection.py
index 909ddb6ef..86abad7ed 100644
--- a/src/context/tests/test_connection.py
+++ b/src/context/tests/test_connection.py
@@ -19,13 +19,12 @@ from context.client.ContextClient import ContextClient
from context.service.database.uuids.Connection import connection_get_uuid
from context.service.database.uuids.EndPoint import endpoint_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import (
CONNECTION_R1_R3, CONNECTION_R1_R3_ID, CONNECTION_R1_R3_NAME, CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID,
DEVICE_R2, DEVICE_R2_ID, DEVICE_R3, DEVICE_R3_ID, SERVICE_R1_R2, SERVICE_R1_R2_ID, SERVICE_R1_R3, SERVICE_R1_R3_ID,
SERVICE_R2_R3, SERVICE_R2_R3_ID, TOPOLOGY, TOPOLOGY_ID)
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_service.py::test_service', 'context/tests/test_slice.py::test_slice'])
def test_connection(context_client : ContextClient) -> None:
diff --git a/src/context/tests/test_context.py b/src/context/tests/test_context.py
index 77f1dc380..7a9564df6 100644
--- a/src/context/tests/test_context.py
+++ b/src/context/tests/test_context.py
@@ -17,10 +17,9 @@ from common.proto.context_pb2 import Context, ContextEvent, ContextId, Empty, Ev
from context.client.ContextClient import ContextClient
from context.service.database.uuids.Context import context_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import CONTEXT, CONTEXT_ID, CONTEXT_NAME
-GET_EVENTS_TIMEOUT = 10.0
-
def test_context(context_client : ContextClient) -> None:
# ----- Initialize the EventsCollector -----------------------------------------------------------------------------
diff --git a/src/context/tests/test_device.py b/src/context/tests/test_device.py
index bcbe4cc3b..615ebe0be 100644
--- a/src/context/tests/test_device.py
+++ b/src/context/tests/test_device.py
@@ -19,10 +19,9 @@ from common.proto.context_pb2 import (
from context.client.ContextClient import ContextClient
from context.service.database.uuids.Device import device_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID, DEVICE_R1_NAME, TOPOLOGY, TOPOLOGY_ID
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_topology.py::test_topology'])
def test_device(context_client : ContextClient) -> None:
diff --git a/src/context/tests/test_link.py b/src/context/tests/test_link.py
index c8ed1d486..e56a1889d 100644
--- a/src/context/tests/test_link.py
+++ b/src/context/tests/test_link.py
@@ -19,12 +19,11 @@ from common.proto.context_pb2 import (
from context.client.ContextClient import ContextClient
from context.client.EventsCollector import EventsCollector
from context.service.database.uuids.Link import link_get_uuid
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import (
CONTEXT, CONTEXT_ID, DEVICE_R1, DEVICE_R1_ID, DEVICE_R2, DEVICE_R2_ID, LINK_R1_R2, LINK_R1_R2_ID, LINK_R1_R2_NAME,
TOPOLOGY, TOPOLOGY_ID)
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_device.py::test_device'])
def test_link(context_client : ContextClient) -> None:
diff --git a/src/context/tests/test_policy.py b/src/context/tests/test_policy.py
index f9bf5ef6d..1cc0b9557 100644
--- a/src/context/tests/test_policy.py
+++ b/src/context/tests/test_policy.py
@@ -19,7 +19,7 @@ from context.client.ContextClient import ContextClient
from context.service.database.uuids.PolicuRule import policyrule_get_uuid
from .Objects import POLICYRULE, POLICYRULE_ID, POLICYRULE_NAME
-@pytest.mark.depends(on=['context/tests/test_device.py::test_device', 'context/tests/test_service.py::test_service'])
+@pytest.mark.depends(on=['context/tests/test_connection.py::test_connection'])
def test_policy(context_client : ContextClient):
# ----- Get when the object does not exist -------------------------------------------------------------------------
diff --git a/src/context/tests/test_service.py b/src/context/tests/test_service.py
index 4e46c24ad..ca02a4a91 100644
--- a/src/context/tests/test_service.py
+++ b/src/context/tests/test_service.py
@@ -19,12 +19,11 @@ from common.proto.context_pb2 import (
from context.client.ContextClient import ContextClient
from context.service.database.uuids.Service import service_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import (
CONTEXT, CONTEXT_ID, CONTEXT_NAME, DEVICE_R1, DEVICE_R1_ID, SERVICE_R1_R2_NAME, DEVICE_R2, DEVICE_R2_ID,
SERVICE_R1_R2, SERVICE_R1_R2_ID, TOPOLOGY, TOPOLOGY_ID)
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_link.py::test_link'])
def test_service(context_client : ContextClient) -> None:
diff --git a/src/context/tests/test_slice.py b/src/context/tests/test_slice.py
index 6996bb39e..1008e7e91 100644
--- a/src/context/tests/test_slice.py
+++ b/src/context/tests/test_slice.py
@@ -19,13 +19,12 @@ from common.proto.context_pb2 import (
from context.client.ContextClient import ContextClient
from context.service.database.uuids.Slice import slice_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import (
CONTEXT, CONTEXT_ID, CONTEXT_NAME, DEVICE_R1, DEVICE_R1_ID, DEVICE_R2, DEVICE_R2_ID, DEVICE_R3, DEVICE_R3_ID,
LINK_R1_R2, LINK_R1_R2_ID, LINK_R1_R3, LINK_R1_R3_ID, LINK_R2_R3, LINK_R2_R3_ID, SERVICE_R1_R2, SERVICE_R1_R2_ID,
SERVICE_R2_R3, SERVICE_R2_R3_ID, SLICE_R1_R3, SLICE_R1_R3_ID, SLICE_R1_R3_NAME, TOPOLOGY, TOPOLOGY_ID)
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_service.py::test_service'])
def test_slice(context_client : ContextClient) -> None:
diff --git a/src/context/tests/test_topology.py b/src/context/tests/test_topology.py
index 6a3367d49..0d8b8c027 100644
--- a/src/context/tests/test_topology.py
+++ b/src/context/tests/test_topology.py
@@ -18,10 +18,9 @@ from common.proto.context_pb2 import (
from context.client.ContextClient import ContextClient
from context.service.database.uuids.Topology import topology_get_uuid
from context.client.EventsCollector import EventsCollector
+from .Constants import GET_EVENTS_TIMEOUT
from .Objects import CONTEXT, CONTEXT_ID, CONTEXT_NAME, TOPOLOGY, TOPOLOGY_ID, TOPOLOGY_NAME
-GET_EVENTS_TIMEOUT = 10.0
-
@pytest.mark.depends(on=['context/tests/test_context.py::test_context'])
def test_topology(context_client : ContextClient) -> None:
--
GitLab