From f986805dda6e3edcee224ad726e3f9d160285e36 Mon Sep 17 00:00:00 2001 From: hajipour Date: Mon, 5 Aug 2024 13:31:57 +0200 Subject: [PATCH 01/16] QoSProfile protobuf update: - QoSProfile messages and rpcs added to context.proto for QoSProfile database support - qos_profile.proto added for QoSProfile component --- proto/context.proto | 37 +++++++++++++++++++++++++++++++++++++ proto/qos_profile.proto | 26 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 proto/qos_profile.proto diff --git a/proto/context.proto b/proto/context.proto index 87f69132d..d23a46a48 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -68,6 +68,12 @@ service ContextService { rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} rpc SelectSlice (SliceFilter ) returns ( SliceList ) {} + rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {} + rpc DeleteQoSProfile (QoSProfileId ) returns (Empty ) {} + rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {} + rpc GetQoSProfiles (Empty ) returns (stream QoSProfile) {} + rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} rpc GetConnection (ConnectionId ) returns ( Connection ) {} @@ -402,6 +408,37 @@ message SliceEvent { SliceId slice_id = 2; } + +// ----- QoSProfile ---------------------------------------------------------------------------------------------------- +message QoSProfileId { + Uuid qos_profile_id = 1; +} + +message QoSProfileValueUnitPair { + int32 value = 1; + string unit = 2; +} + +message QoSProfile { + QoSProfileId qos_profile_id = 1; + string name = 2; + string description = 3; + string status = 4; + QoSProfileValueUnitPair targetMinUpstreamRate = 5; + QoSProfileValueUnitPair maxUpstreamRate = 6; + QoSProfileValueUnitPair maxUpstreamBurstRate = 7; + QoSProfileValueUnitPair targetMinDownstreamRate = 8; + QoSProfileValueUnitPair maxDownstreamRate = 9; + QoSProfileValueUnitPair maxDownstreamBurstRate = 10; + QoSProfileValueUnitPair minDuration = 11; + QoSProfileValueUnitPair maxDuration = 12; + int32 priority = 13; + QoSProfileValueUnitPair packetDelayBudget = 14; + QoSProfileValueUnitPair jitter = 15; + int32 packetErrorLossRate = 16; +} + + // ----- Connection ---------------------------------------------------------------------------------------------------- message ConnectionId { Uuid connection_uuid = 1; diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto new file mode 100644 index 000000000..8e1fc80a3 --- /dev/null +++ b/proto/qos_profile.proto @@ -0,0 +1,26 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package qos_profile; + +import "context.proto"; + +service QoSProfileService { + rpc CreateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} + rpc UpdateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId) returns (context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId) returns (context.QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream context.QoSProfile) {} +} -- GitLab From 8772ed2d02349cfa79580cf5655059c49fe2eed0 Mon Sep 17 00:00:00 2001 From: hajipour Date: Tue, 6 Aug 2024 22:52:48 +0200 Subject: [PATCH 02/16] feat: QoSProfile database and crud operations added to context - QoSProfile port address added to Constants.py - database QoSProfile and sqlalchemy crud operations added - QoSProfile support added to context's server implementation and client --- src/common/Constants.py | 2 + src/context/client/ContextClient.py | 37 ++++++- .../service/ContextServiceServicerImpl.py | 26 ++++- src/context/service/database/QoSProfile.py | 97 +++++++++++++++++++ .../service/database/models/QoSProfile.py | 43 ++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 src/context/service/database/QoSProfile.py create mode 100644 src/context/service/database/models/QoSProfile.py diff --git a/src/common/Constants.py b/src/common/Constants.py index 767b21343..8b0fa80c1 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -65,6 +65,7 @@ class ServiceNameEnum(Enum): KPIVALUEAPI = 'kpi-value-api' KPIVALUEWRITER = 'kpi-value-writer' TELEMETRYFRONTEND = 'telemetry-frontend' + QOSPROFILE = 'qos-profile' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -98,6 +99,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.KPIVALUEAPI .value : 30020, ServiceNameEnum.KPIVALUEWRITER .value : 30030, ServiceNameEnum.TELEMETRYFRONTEND .value : 30050, + ServiceNameEnum.QOSPROFILE .value : 30060, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, diff --git a/src/context/client/ContextClient.py b/src/context/client/ContextClient.py index 2776a0d29..024381c54 100644 --- a/src/context/client/ContextClient.py +++ b/src/context/client/ContextClient.py @@ -27,7 +27,7 @@ from common.proto.context_pb2 import ( Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList, Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList, Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList, - OpticalConfig, OpticalConfigId, OpticalConfigList + OpticalConfig, OpticalConfigId, OpticalConfigList, QoSProfileId, QoSProfile ) from common.proto.context_pb2_grpc import ContextServiceStub from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub @@ -362,6 +362,41 @@ class ContextClient: LOGGER.debug('GetSliceEvents result: {:s}'.format(grpc_message_to_json_string(response))) return response + @RETRY_DECORATOR + def CreateQoSProfile(self, request : QoSProfile) -> QoSProfile: + LOGGER.debug('CreateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.CreateQoSProfile(request) + LOGGER.debug('CreateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def UpdateQoSProfile(self, request : QoSProfile) -> QoSProfile: + LOGGER.debug('UpdateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.UpdateQoSProfile(request) + LOGGER.debug('UpdateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteQoSProfile(self, request : QoSProfileId) -> Empty: + LOGGER.debug('DeleteQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteQoSProfile(request) + LOGGER.debug('DeleteQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfile(self, request : QoSProfileId) -> QoSProfile: + LOGGER.debug('GetQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfile(request) + LOGGER.debug('GetQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfiles(self, request : Empty) -> Iterator[QoSProfile]: + LOGGER.debug('GetQoSProfiles request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfiles(request) + LOGGER.debug('GetQoSProfiles result: {:s}'.format(grpc_message_to_json_string(response))) + return response + @RETRY_DECORATOR def ListConnectionIds(self, request: ServiceId) -> ConnectionIdList: LOGGER.debug('ListConnectionIds request: {:s}'.format(grpc_message_to_json_string(request))) diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index be3237210..ee4bc4de6 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -24,7 +24,7 @@ from common.proto.context_pb2 import ( Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList, Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList, Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList, - OpticalConfigList, OpticalConfigId, OpticalConfig + OpticalConfigList, OpticalConfigId, OpticalConfig, QoSProfileId, QoSProfile ) from common.proto.policy_pb2 import PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule from common.proto.context_pb2_grpc import ContextServiceServicer @@ -46,6 +46,7 @@ from .database.Slice import ( from .database.Topology import ( topology_delete, topology_get, topology_get_details, topology_list_ids, topology_list_objs, topology_set) from .database.OpticalConfig import set_opticalconfig, select_opticalconfig, get_opticalconfig +from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles LOGGER = logging.getLogger(__name__) @@ -251,6 +252,29 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer for message in consume_events(self.messagebroker, {EventTopicEnum.SLICE}): yield message + # ----- QoSProfile ----------------------------------------------------------------------------------------------- + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def CreateQoSProfile(self, request : QoSProfile, context : grpc.ServicerContext) -> QoSProfile: + return set_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def UpdateQoSProfile(self, request : QoSProfile, context : grpc.ServicerContext) -> QoSProfile: + return set_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> Empty: + return delete_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> QoSProfile: + return get_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfiles(self, request : Empty, context : grpc.ServicerContext) -> Iterator[QoSProfile]: + yield from get_qos_profiles(self.db_engine, request) + + # ----- Connection ------------------------------------------------------------------------------------------------- @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) diff --git a/src/context/service/database/QoSProfile.py b/src/context/service/database/QoSProfile.py new file mode 100644 index 000000000..8b50f64de --- /dev/null +++ b/src/context/service/database/QoSProfile.py @@ -0,0 +1,97 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy_cockroachdb import run_transaction +from typing import List, Optional + +from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfileValueUnitPair, QoSProfile +from common.method_wrappers.ServiceExceptions import NotFoundException +from common.tools.grpc.Tools import grpc_message_to_json +from .models.QoSProfile import QoSProfileModel + +LOGGER = logging.getLogger(__name__) + +def grpc_message_to_qos_table_data(message: QoSProfile) -> dict: + return [{ + 'qos_profile_id' : message.qos_profile_id.uuid, + 'name' : message.name, + 'description' : message.description, + 'status' : message.status, + 'targetMinUpstreamRate' : grpc_message_to_json(message.targetMinUpstreamRate), + 'maxUpstreamRate' : grpc_message_to_json(message.maxUpstreamRate), + 'maxUpstreamBurstRate' : grpc_message_to_json(message.maxUpstreamBurstRate), + 'targetMinDownstreamRate' : grpc_message_to_json(message.targetMinDownstreamRate), + 'maxDownstreamRate' : grpc_message_to_json(message.maxDownstreamRate), + 'maxDownstreamBurstRate' : grpc_message_to_json(message.maxDownstreamBurstRate), + 'minDuration' : grpc_message_to_json(message.minDuration), + 'maxDuration' : grpc_message_to_json(message.maxDuration), + 'priority' : message.priority, + 'packetDelayBudget' : grpc_message_to_json(message.packetDelayBudget), + 'jitter' : grpc_message_to_json(message.jitter), + 'packetErrorLossRate' : message.packetErrorLossRate, + }] + +def qos_table_data_to_grpc_message(data: QoSProfileModel) -> QoSProfile: + QoSProfile( + qos_profile_id = QoSProfileId(uuid=data.qos_profile_id), + name = data.name, + description = data.description, + status = data.status, + targetMinUpstreamRate = QoSProfileValueUnitPair(**data.targetMinUpstreamRate), + maxUpstreamRate = QoSProfileValueUnitPair(**data.maxUpstreamRate), + maxUpstreamBurstRate = QoSProfileValueUnitPair(**data.maxUpstreamBurstRate), + targetMinDownstreamRate = QoSProfileValueUnitPair(**data.targetMinDownstreamRate), + maxDownstreamRate = QoSProfileValueUnitPair(**data.maxDownstreamRate), + maxDownstreamBurstRate = QoSProfileValueUnitPair(**data.maxDownstreamBurstRate), + minDuration = QoSProfileValueUnitPair(**data.minDuration), + maxDuration = QoSProfileValueUnitPair(**data.maxDuration), + priority = data.priority, + packetDelayBudget = QoSProfileValueUnitPair(**data.packetDelayBudget), + jitter = QoSProfileValueUnitPair(**data.jitter), + packetErrorLossRate = data.packetErrorLossRate + ) + +def set_qos_profile(db_engine : Engine, request : QoSProfile) -> QoSProfile: + qos_profile_data = grpc_message_to_qos_table_data(request) + def callback(session : Session) -> bool: + stmt = insert(QoSProfileModel).values(qos_profile_data) + session.execute(stmt) + return get_qos_profile(db_engine, request.qos_profile_id.uuid) + return run_transaction(sessionmaker(bind=db_engine), callback) + +def delete_qos_profile(db_engine : Engine, request : str) -> Empty: + def callback(session : Session) -> bool: + num_deleted = session.query(QoSProfileModel).filter_by(qos_profile_id=request).delete() + return num_deleted > 0 + deleted = run_transaction(sessionmaker(bind=db_engine), callback) + return Empty() + +def get_qos_profile(db_engine : Engine, request : str) -> QoSProfile: + def callback(session : Session) -> Optional[QoSProfile]: + obj : Optional[QoSProfileModel] = session.query(QoSProfileModel).filter_by(qos_profile_id=request).one_or_none() + return None if obj is None else qos_table_data_to_grpc_message(obj) + qos_profile = run_transaction(sessionmaker(bind=db_engine), callback) + if qos_profile is None: + raise NotFoundException('QoSProfile', request) + return qos_profile + +def get_qos_profiles(db_engine : Engine, request : Empty) -> List[QoSProfile]: + def callback(session : Session) -> List[QoSProfile]: + obj_list : List[QoSProfileModel] = session.query(QoSProfileModel).all() + return [qos_table_data_to_grpc_message(obj) for obj in obj_list] + return run_transaction(sessionmaker(bind=db_engine), callback) diff --git a/src/context/service/database/models/QoSProfile.py b/src/context/service/database/models/QoSProfile.py new file mode 100644 index 000000000..0ce7365aa --- /dev/null +++ b/src/context/service/database/models/QoSProfile.py @@ -0,0 +1,43 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy import Column, Integer, String, JSON +from sqlalchemy.dialects.postgresql import UUID +from typing import TypedDict +from ._Base import _Base + + +class QoSProfileValueUnitPair(TypedDict): + value: int + unit: str + +class QoSProfileModel(_Base): + __tablename__ = 'qos_profile' + + qos_profile_id = Column(UUID(as_uuid=False), primary_key=True) + name = Column(String, nullable=False) + description = Column(String, nullable=False) + status = Column(String, nullable=False) + targetMinUpstreamRate = Column(JSON, nullable=False) + maxUpstreamRate = Column(JSON, nullable=False) + maxUpstreamBurstRate = Column(JSON, nullable=False) + targetMinDownstreamRate = Column(JSON, nullable=False) + maxDownstreamRate = Column(JSON, nullable=False) + maxDownstreamBurstRate = Column(JSON, nullable=False) + minDuration = Column(JSON, nullable=False) + maxDuration = Column(JSON, nullable=False) + priority = Column(Integer, nullable=False) + packetDelayBudget = Column(JSON, nullable=False) + jitter = Column(JSON, nullable=False) + packetErrorLossRate = Column(Integer, nullable=False) -- GitLab From 874f8c3f292396235c5f6ca9266b5307bc71b478 Mon Sep 17 00:00:00 2001 From: hajipour Date: Wed, 7 Aug 2024 22:55:32 +0200 Subject: [PATCH 03/16] feat: QoSProfile component added: - manifest added - debug in QoSProfile-related contextservice methods - debug in QoSProfile crud operations and changing id to string - Dockerfile added - integration tests added --- manifests/qos_profileservice.yaml | 72 ++++++++++ .../service/ContextServiceServicerImpl.py | 4 +- src/context/service/database/QoSProfile.py | 42 ++++-- .../service/database/models/QoSProfile.py | 8 +- src/qos_profile/Config.py | 14 ++ src/qos_profile/Dockerfile | 76 ++++++++++ src/qos_profile/__init__.py | 14 ++ src/qos_profile/client/QoSProfileClient.py | 82 +++++++++++ src/qos_profile/client/__init__.py | 14 ++ src/qos_profile/requirements.in | 15 ++ src/qos_profile/service/QoSProfileService.py | 28 ++++ .../service/QoSProfileServiceServicerImpl.py | 77 +++++++++++ src/qos_profile/service/__init__.py | 14 ++ src/qos_profile/service/__main__.py | 66 +++++++++ src/qos_profile/tests/.gitignore | 1 + src/qos_profile/tests/__init__.py | 14 ++ src/qos_profile/tests/conftest.py | 23 ++++ src/qos_profile/tests/test_crud.py | 130 ++++++++++++++++++ 18 files changed, 675 insertions(+), 19 deletions(-) create mode 100644 manifests/qos_profileservice.yaml create mode 100644 src/qos_profile/Config.py create mode 100644 src/qos_profile/Dockerfile create mode 100644 src/qos_profile/__init__.py create mode 100644 src/qos_profile/client/QoSProfileClient.py create mode 100644 src/qos_profile/client/__init__.py create mode 100644 src/qos_profile/requirements.in create mode 100644 src/qos_profile/service/QoSProfileService.py create mode 100644 src/qos_profile/service/QoSProfileServiceServicerImpl.py create mode 100644 src/qos_profile/service/__init__.py create mode 100644 src/qos_profile/service/__main__.py create mode 100644 src/qos_profile/tests/.gitignore create mode 100644 src/qos_profile/tests/__init__.py create mode 100644 src/qos_profile/tests/conftest.py create mode 100644 src/qos_profile/tests/test_crud.py diff --git a/manifests/qos_profileservice.yaml b/manifests/qos_profileservice.yaml new file mode 100644 index 000000000..1bcaa500f --- /dev/null +++ b/manifests/qos_profileservice.yaml @@ -0,0 +1,72 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: qos-profileservice +spec: + selector: + matchLabels: + app: qos-profileservice + #replicas: 1 + template: + metadata: + labels: + app: qos-profileservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/qos_profile:latest + imagePullPolicy: Always + ports: + - containerPort: 30060 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: qos-profileservice + labels: + app: qos-profileservice +spec: + type: ClusterIP + selector: + app: qos-profileservice + ports: + - name: grpc + protocol: TCP + port: 30060 + targetPort: 30060 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index ee4bc4de6..414548000 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -264,11 +264,11 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> Empty: - return delete_qos_profile(self.db_engine, request) + return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> QoSProfile: - return get_qos_profile(self.db_engine, request) + return get_qos_profile(self.db_engine, request.qos_profile_id.uuid) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfiles(self, request : Empty, context : grpc.ServicerContext) -> Iterator[QoSProfile]: diff --git a/src/context/service/database/QoSProfile.py b/src/context/service/database/QoSProfile.py index 8b50f64de..8ffe8e0ec 100644 --- a/src/context/service/database/QoSProfile.py +++ b/src/context/service/database/QoSProfile.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_cockroachdb import run_transaction from typing import List, Optional -from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfileValueUnitPair, QoSProfile +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId, QoSProfileValueUnitPair, QoSProfile from common.method_wrappers.ServiceExceptions import NotFoundException from common.tools.grpc.Tools import grpc_message_to_json from .models.QoSProfile import QoSProfileModel @@ -27,8 +27,8 @@ from .models.QoSProfile import QoSProfileModel LOGGER = logging.getLogger(__name__) def grpc_message_to_qos_table_data(message: QoSProfile) -> dict: - return [{ - 'qos_profile_id' : message.qos_profile_id.uuid, + return { + 'qos_profile_id' : message.qos_profile_id.qos_profile_id.uuid, 'name' : message.name, 'description' : message.description, 'status' : message.status, @@ -44,11 +44,11 @@ def grpc_message_to_qos_table_data(message: QoSProfile) -> dict: 'packetDelayBudget' : grpc_message_to_json(message.packetDelayBudget), 'jitter' : grpc_message_to_json(message.jitter), 'packetErrorLossRate' : message.packetErrorLossRate, - }] + } def qos_table_data_to_grpc_message(data: QoSProfileModel) -> QoSProfile: - QoSProfile( - qos_profile_id = QoSProfileId(uuid=data.qos_profile_id), + return QoSProfile( + qos_profile_id = QoSProfileId(qos_profile_id=Uuid(uuid=data.qos_profile_id)), name = data.name, description = data.description, status = data.status, @@ -69,10 +69,32 @@ def qos_table_data_to_grpc_message(data: QoSProfileModel) -> QoSProfile: def set_qos_profile(db_engine : Engine, request : QoSProfile) -> QoSProfile: qos_profile_data = grpc_message_to_qos_table_data(request) def callback(session : Session) -> bool: - stmt = insert(QoSProfileModel).values(qos_profile_data) - session.execute(stmt) - return get_qos_profile(db_engine, request.qos_profile_id.uuid) - return run_transaction(sessionmaker(bind=db_engine), callback) + stmt = insert(QoSProfileModel).values([qos_profile_data]) + stmt = stmt.on_conflict_do_update(index_elements=[QoSProfileModel.qos_profile_id], + set_=dict( + + name = stmt.excluded.name, + description = stmt.excluded.description, + status = stmt.excluded.status, + targetMinUpstreamRate = stmt.excluded.targetMinUpstreamRate, + maxUpstreamRate = stmt.excluded.maxUpstreamRate, + maxUpstreamBurstRate = stmt.excluded.maxUpstreamBurstRate, + targetMinDownstreamRate = stmt.excluded.targetMinDownstreamRate, + maxDownstreamRate = stmt.excluded.maxDownstreamRate, + maxDownstreamBurstRate = stmt.excluded.maxDownstreamBurstRate, + minDuration = stmt.excluded.minDuration, + maxDuration = stmt.excluded.maxDuration, + priority = stmt.excluded.priority, + packetDelayBudget = stmt.excluded.packetDelayBudget, + jitter = stmt.excluded.jitter, + packetErrorLossRate = stmt.excluded.packetErrorLossRate, + ) + ) + stmt = stmt.returning(QoSProfileModel) + qos_profile = session.execute(stmt).fetchall() + return qos_profile[0] + qos_profile_row = run_transaction(sessionmaker(bind=db_engine), callback) + return qos_table_data_to_grpc_message(qos_profile_row) def delete_qos_profile(db_engine : Engine, request : str) -> Empty: def callback(session : Session) -> bool: diff --git a/src/context/service/database/models/QoSProfile.py b/src/context/service/database/models/QoSProfile.py index 0ce7365aa..431d0f503 100644 --- a/src/context/service/database/models/QoSProfile.py +++ b/src/context/service/database/models/QoSProfile.py @@ -13,19 +13,13 @@ # limitations under the License. from sqlalchemy import Column, Integer, String, JSON -from sqlalchemy.dialects.postgresql import UUID -from typing import TypedDict from ._Base import _Base -class QoSProfileValueUnitPair(TypedDict): - value: int - unit: str - class QoSProfileModel(_Base): __tablename__ = 'qos_profile' - qos_profile_id = Column(UUID(as_uuid=False), primary_key=True) + qos_profile_id = Column(String, primary_key=True) name = Column(String, nullable=False) description = Column(String, nullable=False) status = Column(String, nullable=False) diff --git a/src/qos_profile/Config.py b/src/qos_profile/Config.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/Config.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile new file mode 100644 index 000000000..a35c2e741 --- /dev/null +++ b/src/qos_profile/Dockerfile @@ -0,0 +1,76 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/qos_profile +WORKDIR /var/teraflow/qos_profile +COPY src/service/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py +COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/qos_profile/. qos_profile/ + +# Start the service +ENTRYPOINT ["python", "-m", "qos_profile.service"] diff --git a/src/qos_profile/__init__.py b/src/qos_profile/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py new file mode 100644 index 000000000..7a857ec25 --- /dev/null +++ b/src/qos_profile/client/QoSProfileClient.py @@ -0,0 +1,82 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterator +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile +from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class QoSProfileClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.QOSPROFILE) + if not port: port = get_service_port_grpc(ServiceNameEnum.QOSPROFILE) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = QoSProfileServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def CreateQoSProfile(self, request: QoSProfile) -> QoSProfile: + LOGGER.debug('CreateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.CreateQoSProfile(request) + LOGGER.debug('CreateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def UpdateQoSProfile(self, request: QoSProfile) -> QoSProfile: + LOGGER.debug('UpdateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.UpdateQoSProfile(request) + LOGGER.debug('UpdateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteQoSProfile(self, request: QoSProfileId) -> Empty: + LOGGER.debug('DeleteQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteQoSProfile(request) + LOGGER.debug('DeleteQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfile(self, request: QoSProfileId) -> QoSProfile: + LOGGER.debug('GetQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfile(request) + LOGGER.debug('GetQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfiles(self, request: Empty) -> Iterator[QoSProfile]: + LOGGER.debug('GetQoSProfiles request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfiles(request) + LOGGER.debug('GetQoSProfiles result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/qos_profile/client/__init__.py b/src/qos_profile/client/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/requirements.in b/src/qos_profile/requirements.in new file mode 100644 index 000000000..5cf553eaa --- /dev/null +++ b/src/qos_profile/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/qos_profile/service/QoSProfileService.py b/src/qos_profile/service/QoSProfileService.py new file mode 100644 index 000000000..bdc90f5bc --- /dev/null +++ b/src/qos_profile/service/QoSProfileService.py @@ -0,0 +1,28 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.qos_profile_pb2_grpc import add_QoSProfileServiceServicer_to_server +from common.tools.service.GenericGrpcService import GenericGrpcService +from .QoSProfileServiceServicerImpl import QoSProfileServiceServicerImpl + +class QoSProfileService(GenericGrpcService): + def __init__(self, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.QOSPROFILE) + super().__init__(port, cls_name=cls_name) + self.qos_profile_servicer = QoSProfileServiceServicerImpl() + + def install_servicers(self): + add_QoSProfileServiceServicer_to_server(self.qos_profile_servicer, self.server) diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py new file mode 100644 index 000000000..38ad608d1 --- /dev/null +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -0,0 +1,77 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from typing import Iterator +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.method_wrappers.ServiceExceptions import AlreadyExistsException +from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile +from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer +from context.client.ContextClient import ContextClient + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('QoSProfile', 'RPC') + +class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): + def __init__(self ) -> None: + LOGGER.debug('Servicer Created') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: + context_client = ContextClient() + try: + qos_profile_get = context_client.GetQoSProfile(request.qos_profile_id) + except grpc.RpcError: + qos_profile_get = None + if qos_profile_get: + raise AlreadyExistsException('QoSProfile', request.qos_profile_id) + qos_profile = context_client.CreateQoSProfile(request) + return qos_profile + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: + context_client = ContextClient() + try: + _ = context_client.GetQoSProfile(request.qos_profile_id) + except grpc.RpcError as exc: + raise exc + qos_profile = context_client.UpdateQoSProfile(request) + return qos_profile + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty: + context_client = ContextClient() + try: + _ = context_client.GetQoSProfile(request) + except grpc.RpcError as exc: + raise exc + empty = context_client.DeleteQoSProfile(request) + return empty + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> QoSProfile: + context_client = ContextClient() + try: + qos_profile = context_client.GetQoSProfile(request) + except grpc.RpcError as exc: + raise exc + return qos_profile + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfiles(self, request: Empty, context: grpc.ServicerContext) -> Iterator[QoSProfile]: + context_client = ContextClient() + yield from context_client.GetQoSProfiles(request) + diff --git a/src/qos_profile/service/__init__.py b/src/qos_profile/service/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py new file mode 100644 index 000000000..ccd1ca23f --- /dev/null +++ b/src/qos_profile/service/__main__.py @@ -0,0 +1,66 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from prometheus_client import start_http_server +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, + wait_for_environment_variables +) +from .QoSProfileService import QoSProfileService + +terminate = threading.Event() +LOGGER : logging.Logger = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting service service + grpc_service = QoSProfileService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/qos_profile/tests/.gitignore b/src/qos_profile/tests/.gitignore new file mode 100644 index 000000000..6b97d6fe3 --- /dev/null +++ b/src/qos_profile/tests/.gitignore @@ -0,0 +1 @@ +# Add here your files containing confidential testbed details such as IP addresses, ports, usernames, passwords, etc. diff --git a/src/qos_profile/tests/__init__.py b/src/qos_profile/tests/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py new file mode 100644 index 000000000..37110c0b6 --- /dev/null +++ b/src/qos_profile/tests/conftest.py @@ -0,0 +1,23 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from qos_profile.client.QoSProfileClient import QoSProfileClient + +@pytest.fixture(scope='function') +def qos_profile_client(): + _client = QoSProfileClient(host='0.0.0.0', port=30060) + yield _client + _client.close() + diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py new file mode 100644 index 000000000..1c5171da0 --- /dev/null +++ b/src/qos_profile/tests/test_crud.py @@ -0,0 +1,130 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from grpc import RpcError +import logging, pytest +from common.proto.context_pb2 import Empty, Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile + +from common.tools.grpc.Tools import grpc_message_to_json_string +from qos_profile.client.QoSProfileClient import QoSProfileClient + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +qos_profile_data = { + "qos_profile_id": "f00406f5-8e36-4abc-a0ec-b871c7f062b7", + "name": "QCI_1_voice", + "description": "QoS profile for video streaming", + "status": "ACTIVE", + "targetMinUpstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxUpstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxUpstreamBurstRate": { + "value": 10, + "unit": "bps" + }, + "targetMinDownstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxDownstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxDownstreamBurstRate": { + "value": 10, + "unit": "bps" + }, + "minDuration": { + "value": 12, + "unit": "Minutes" + }, + "maxDuration": { + "value": 12, + "unit": "Minutes" + }, + "priority": 20, + "packetDelayBudget": { + "value": 12, + "unit": "Minutes" + }, + "jitter": { + "value": 12, + "unit": "Minutes" + }, + "packetErrorLossRate": 3 +} + +def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: + def create_QoSProfileValueUnitPair(data) -> QoSProfileValueUnitPair: + return QoSProfileValueUnitPair(value=data['value'], unit=data['unit']) + qos_profile = QoSProfile() + qos_profile.qos_profile_id.CopyFrom(QoSProfileId(qos_profile_id=Uuid(uuid=qos_profile_data['qos_profile_id']))) + qos_profile.name = qos_profile_data['name'] + qos_profile.description = qos_profile_data['description'] + qos_profile.status = qos_profile_data['status'] + qos_profile.targetMinUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinUpstreamRate'])) + qos_profile.maxUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamRate'])) + qos_profile.maxUpstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamBurstRate'])) + qos_profile.targetMinDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinDownstreamRate'])) + qos_profile.maxDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamRate'])) + qos_profile.maxDownstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamBurstRate'])) + qos_profile.minDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['minDuration'])) + qos_profile.maxDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDuration'])) + qos_profile.priority = qos_profile_data['priority'] + qos_profile.packetDelayBudget.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['packetDelayBudget'])) + qos_profile.jitter.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['jitter'])) + qos_profile.packetErrorLossRate = qos_profile_data['packetErrorLossRate'] + return qos_profile + +def test_create_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_created))) + assert qos_profile == qos_profile_created + + +def test_failed_create_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + with pytest.raises(RpcError) as exc: + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + +def test_get_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_got = qos_profile_client.GetQoSProfile(qos_profile.qos_profile_id) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_got))) + assert qos_profile == qos_profile_got + +def test_get_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty())) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got))) + assert qos_profile == qos_profiles_got[0] + +def test_update_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile.packetErrorLossRate = 5 + qos_profile_updated = qos_profile_client.UpdateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_updated))) + assert qos_profile_updated.packetErrorLossRate == 5 + +def test_delete_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + with pytest.raises(RpcError) as exc: + qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(QoSProfileId(qos_profile_id=Uuid(uuid='f8b1c625-ac01-405c-b1f7-b5ee06e16282'))) \ No newline at end of file -- GitLab From 6408e217e8e30c1a86695c2dac3d807731a96379 Mon Sep 17 00:00:00 2001 From: hajipour Date: Thu, 8 Aug 2024 11:00:10 +0200 Subject: [PATCH 04/16] feat: .gitlab-ci.yml added for QoSProfile component --- .gitlab-ci.yml | 1 + src/qos_profile/.gitlab-ci.yml | 151 +++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 src/qos_profile/.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c5ff9325..5f826eadb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,7 @@ include: - local: '/src/kpi_manager/.gitlab-ci.yml' - local: '/src/kpi_value_api/.gitlab-ci.yml' - local: '/src/kpi_value_writer/.gitlab-ci.yml' + - local: '/src/qos_profile/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml new file mode 100644 index 000000000..542903b29 --- /dev/null +++ b/src/qos_profile/.gitlab-ci.yml @@ -0,0 +1,151 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build service: + variables: + IMAGE_NAME: "qos_profile" # name of the microservice + IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test qos_profile: + variables: + IMAGE_NAME: "qos_profile" # name of the microservice + IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build qos_profile + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + + # Context-related + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + + # Context-related + - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + + # QoSProfile-related + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + + script: + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker pull "nats:2.9" + - docker pull "$CI_REGISTRY_IMAGE/context:$IMAGE_TAG" + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + + # Context preparation + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - > + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 + nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + - docker logs nats + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $NATS_ADDRESS + - > + docker run --name context -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "MB_BACKEND=nats" + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CONTEXTSERVICE_SERVICE_HOST + + # QoSProfile preparation + - > + docker run --name $IMAGE_NAME -d -p 3030:3030 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + + # Check status before the tests + - sleep 5 + - docker ps -a + - docker logs context + - docker logs $IMAGE_NAME + + # Run the tests + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_crud.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + # Check status after the tests + - docker ps -a + - docker logs context + - docker logs $IMAGE_NAME + + - docker rm -f $IMAGE_NAME + - docker rm -f context + + - docker rm -f $IMAGE_NAME crdb nats + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml -- GitLab From 276a8d44c68f8acb2d0f55779dc3846c9d743b06 Mon Sep 17 00:00:00 2001 From: hajipour Date: Thu, 8 Aug 2024 11:14:02 +0200 Subject: [PATCH 05/16] debug: QoSProfile gitlab-ci.yml bug resolved --- src/qos_profile/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml index 542903b29..29ac9f3b4 100644 --- a/src/qos_profile/.gitlab-ci.yml +++ b/src/qos_profile/.gitlab-ci.yml @@ -13,7 +13,7 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -build service: +build qos_profile: variables: IMAGE_NAME: "qos_profile" # name of the microservice IMAGE_TAG: "latest" # tag of the container image (production, development, etc) -- GitLab From 291f08027458cb6455f85cde0b5f899a69087442 Mon Sep 17 00:00:00 2001 From: hajipour Date: Thu, 8 Aug 2024 12:22:45 +0200 Subject: [PATCH 06/16] feat: QoSProfile error handling added: - not found and already exists errors added to server implementations - test_crud.py updated --- .../service/ContextServiceServicerImpl.py | 7 +++- src/context/service/database/QoSProfile.py | 7 ++-- .../service/QoSProfileServiceServicerImpl.py | 34 +++++++++++++------ src/qos_profile/tests/test_crud.py | 13 +++++-- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index 414548000..e6c305f2f 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -268,7 +268,12 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> QoSProfile: - return get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return qos_profile @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfiles(self, request : Empty, context : grpc.ServicerContext) -> Iterator[QoSProfile]: diff --git a/src/context/service/database/QoSProfile.py b/src/context/service/database/QoSProfile.py index 8ffe8e0ec..33f6cc6ec 100644 --- a/src/context/service/database/QoSProfile.py +++ b/src/context/service/database/QoSProfile.py @@ -103,14 +103,11 @@ def delete_qos_profile(db_engine : Engine, request : str) -> Empty: deleted = run_transaction(sessionmaker(bind=db_engine), callback) return Empty() -def get_qos_profile(db_engine : Engine, request : str) -> QoSProfile: +def get_qos_profile(db_engine : Engine, request : str) -> Optional[QoSProfile]: def callback(session : Session) -> Optional[QoSProfile]: obj : Optional[QoSProfileModel] = session.query(QoSProfileModel).filter_by(qos_profile_id=request).one_or_none() return None if obj is None else qos_table_data_to_grpc_message(obj) - qos_profile = run_transaction(sessionmaker(bind=db_engine), callback) - if qos_profile is None: - raise NotFoundException('QoSProfile', request) - return qos_profile + return run_transaction(sessionmaker(bind=db_engine), callback) def get_qos_profiles(db_engine : Engine, request : Empty) -> List[QoSProfile]: def callback(session : Session) -> List[QoSProfile]: diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index 38ad608d1..cabca909d 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -14,8 +14,9 @@ import grpc, logging from typing import Iterator + +import grpc._channel from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.method_wrappers.ServiceExceptions import AlreadyExistsException from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer from context.client.ContextClient import ContextClient @@ -34,10 +35,12 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): context_client = ContextClient() try: qos_profile_get = context_client.GetQoSProfile(request.qos_profile_id) - except grpc.RpcError: - qos_profile_get = None - if qos_profile_get: - raise AlreadyExistsException('QoSProfile', request.qos_profile_id) + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists') + context.set_code(grpc.StatusCode.ALREADY_EXISTS) + return QoSProfile() + except grpc._channel._InactiveRpcError as exc: + if exc.code() != grpc.StatusCode.NOT_FOUND: + raise exc qos_profile = context_client.CreateQoSProfile(request) return qos_profile @@ -46,8 +49,11 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): context_client = ContextClient() try: _ = context_client.GetQoSProfile(request.qos_profile_id) - except grpc.RpcError as exc: - raise exc + except grpc._channel._InactiveRpcError as exc: + if exc.code() == grpc.StatusCode.NOT_FOUND: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() qos_profile = context_client.UpdateQoSProfile(request) return qos_profile @@ -56,8 +62,11 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): context_client = ContextClient() try: _ = context_client.GetQoSProfile(request) - except grpc.RpcError as exc: - raise exc + except grpc._channel._InactiveRpcError as exc: + if exc.code() == grpc.StatusCode.NOT_FOUND: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() empty = context_client.DeleteQoSProfile(request) return empty @@ -66,8 +75,11 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): context_client = ContextClient() try: qos_profile = context_client.GetQoSProfile(request) - except grpc.RpcError as exc: - raise exc + except grpc._channel._InactiveRpcError as exc: + if exc.code() == grpc.StatusCode.NOT_FOUND: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() return qos_profile @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index 1c5171da0..89bc87322 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc import RpcError +from grpc import RpcError, StatusCode import logging, pytest from common.proto.context_pb2 import Empty, Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile @@ -104,6 +104,7 @@ def test_failed_create_qos_profile(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) with pytest.raises(RpcError) as exc: qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + assert exc.value.code() == StatusCode.ALREADY_EXISTS def test_get_qos_profile(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) @@ -124,7 +125,13 @@ def test_update_qos_profile(qos_profile_client: QoSProfileClient): LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_updated))) assert qos_profile_updated.packetErrorLossRate == 5 -def test_delete_qos_profiles(qos_profile_client: QoSProfileClient): +def test_failed_delete_qos_profiles(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) with pytest.raises(RpcError) as exc: - qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(QoSProfileId(qos_profile_id=Uuid(uuid='f8b1c625-ac01-405c-b1f7-b5ee06e16282'))) \ No newline at end of file + qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(QoSProfileId(qos_profile_id=Uuid(uuid='f8b1c625-ac01-405c-b1f7-b5ee06e16282'))) + assert exc.value.code() == StatusCode.NOT_FOUND + +def test_delete_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) + assert qos_profiles_deleted == Empty() \ No newline at end of file -- GitLab From 90d55e7389a6442ac0a73a9216c7060bc8dd05c3 Mon Sep 17 00:00:00 2001 From: hajipour Date: Mon, 9 Sep 2024 14:11:07 +0200 Subject: [PATCH 07/16] feat: qos profile show logs script added --- scripts/show_logs_qos_profile.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/show_logs_qos_profile.sh diff --git a/scripts/show_logs_qos_profile.sh b/scripts/show_logs_qos_profile.sh new file mode 100755 index 000000000..744bed9a6 --- /dev/null +++ b/scripts/show_logs_qos_profile.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qos_profileservice -c server -- GitLab From 4c479a1b2d921d1fbf8cd39d9f3d847875a9fe96 Mon Sep 17 00:00:00 2001 From: hajipour Date: Wed, 11 Sep 2024 20:26:38 +0200 Subject: [PATCH 08/16] feat: constraints list retrieval added to QoSProfile component: - test_crud.py refactored - test_constraints.py added - start_timestamp of Constraint_Schedule changed to double from float --- proto/context.proto | 35 ++++- proto/qos_profile.proto | 11 +- src/qos_profile/client/QoSProfileClient.py | 9 +- .../service/QoSProfileServiceServicerImpl.py | 35 ++++- src/qos_profile/tests/conftest.py | 24 +++ src/qos_profile/tests/test_constraints.py | 142 ++++++++++++++++++ src/qos_profile/tests/test_crud.py | 26 +--- 7 files changed, 245 insertions(+), 37 deletions(-) create mode 100644 src/qos_profile/tests/test_constraints.py diff --git a/proto/context.proto b/proto/context.proto index d23a46a48..7f99c4525 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -68,11 +68,11 @@ service ContextService { rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} rpc SelectSlice (SliceFilter ) returns ( SliceList ) {} - rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc DeleteQoSProfile (QoSProfileId ) returns (Empty ) {} - rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {} - rpc GetQoSProfiles (Empty ) returns (stream QoSProfile) {} + rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc DeleteQoSProfile (QoSProfileId ) returns ( Empty ) {} + rpc GetQoSProfile (QoSProfileId ) returns ( QoSProfile ) {} + rpc GetQoSProfiles (Empty ) returns (stream QoSProfile ) {} rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} @@ -419,6 +419,12 @@ message QoSProfileValueUnitPair { string unit = 2; } +message QoDConstraintsRequest { + QoSProfileId qos_profile_id = 1; + double start_timestamp = 2; + float duration = 3; +} + message QoSProfile { QoSProfileId qos_profile_id = 1; string name = 2; @@ -571,7 +577,7 @@ message Constraint_Custom { } message Constraint_Schedule { - float start_timestamp = 1; + double start_timestamp = 1; float duration_days = 2; } @@ -634,6 +640,22 @@ message Constraint_Exclusions { repeated LinkId link_ids = 4; } +message Constraint_QoSProfile { + QoSProfileValueUnitPair target_min_upstream_rate = 1; + QoSProfileValueUnitPair max_upstream_rate = 2; + QoSProfileValueUnitPair max_upstream_burst_rate = 3; + QoSProfileValueUnitPair target_min_downstream_rate = 4; + QoSProfileValueUnitPair max_downstream_rate = 5; + QoSProfileValueUnitPair max_downstream_burst_rate = 6; + QoSProfileValueUnitPair min_duration = 7; + QoSProfileValueUnitPair max_duration = 8; + int32 priority = 9; + QoSProfileValueUnitPair packet_delay_budget = 10; + QoSProfileValueUnitPair jitter = 11; + int32 packet_error_loss_rate = 12; + +} + message Constraint { ConstraintActionEnum action = 1; oneof constraint { @@ -646,6 +668,7 @@ message Constraint { Constraint_SLA_Availability sla_availability = 8; Constraint_SLA_Isolation_level sla_isolation = 9; Constraint_Exclusions exclusions = 10; + Constraint_QoSProfile qos_profile = 11; } } diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto index 8e1fc80a3..557cef539 100644 --- a/proto/qos_profile.proto +++ b/proto/qos_profile.proto @@ -18,9 +18,10 @@ package qos_profile; import "context.proto"; service QoSProfileService { - rpc CreateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} - rpc UpdateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} - rpc DeleteQoSProfile (context.QoSProfileId) returns (context.Empty ) {} - rpc GetQoSProfile (context.QoSProfileId) returns (context.QoSProfile ) {} - rpc GetQoSProfiles (context.Empty ) returns (stream context.QoSProfile) {} + rpc CreateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} + rpc UpdateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId ) returns (context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId ) returns (context.QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream context.QoSProfile ) {} + rpc GetConstraintListFromQoSProfile (context.QoDConstraintsRequest) returns (stream context.Constraint ) {} } diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py index 7a857ec25..e7bec8739 100644 --- a/src/qos_profile/client/QoSProfileClient.py +++ b/src/qos_profile/client/QoSProfileClient.py @@ -16,7 +16,7 @@ from typing import Iterator import grpc, logging from common.Constants import ServiceNameEnum from common.Settings import get_service_host, get_service_port_grpc -from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile +from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile, QoDConstraintsRequest, Constraint from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string @@ -80,3 +80,10 @@ class QoSProfileClient: response = self.stub.GetQoSProfiles(request) LOGGER.debug('GetQoSProfiles result: {:s}'.format(grpc_message_to_json_string(response))) return response + + @RETRY_DECORATOR + def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest) -> Iterator[Constraint]: + LOGGER.debug('GetConstraintListFromQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetConstraintListFromQoSProfile(request) + LOGGER.debug('GetConstraintListFromQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response \ No newline at end of file diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index cabca909d..bdcc3e8c3 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -17,7 +17,7 @@ from typing import Iterator import grpc._channel from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile +from common.proto.context_pb2 import QoDConstraintsRequest, Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId, QoSProfile from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer from context.client.ContextClient import ContextClient @@ -87,3 +87,36 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): context_client = ContextClient() yield from context_client.GetQoSProfiles(request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]: + context_client = ContextClient() + try: + qos_profile = context_client.GetQoSProfile(request.qos_profile_id) + except grpc._channel._InactiveRpcError as exc: + if exc.code() == grpc.StatusCode.NOT_FOUND: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + yield Constraint() + + qos_profile_constraint = Constraint_QoSProfile() + qos_profile_constraint.target_min_upstream_rate.CopyFrom(qos_profile.targetMinUpstreamRate) + qos_profile_constraint.max_upstream_rate.CopyFrom(qos_profile.maxUpstreamRate) + qos_profile_constraint.max_upstream_burst_rate.CopyFrom(qos_profile.maxUpstreamBurstRate) + qos_profile_constraint.target_min_downstream_rate.CopyFrom(qos_profile.targetMinDownstreamRate) + qos_profile_constraint.max_downstream_rate.CopyFrom(qos_profile.maxDownstreamRate) + qos_profile_constraint.max_downstream_burst_rate.CopyFrom(qos_profile.maxDownstreamBurstRate) + qos_profile_constraint.min_duration.CopyFrom(qos_profile.minDuration) + qos_profile_constraint.max_duration.CopyFrom(qos_profile.maxDuration) + qos_profile_constraint.priority = qos_profile.priority + qos_profile_constraint.packet_delay_budget.CopyFrom(qos_profile.packetDelayBudget) + qos_profile_constraint.jitter.CopyFrom(qos_profile.jitter) + qos_profile_constraint.packet_error_loss_rate =qos_profile.packetErrorLossRate + constraint_qos = Constraint() + constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET + constraint_qos.qos_profile.CopyFrom(qos_profile_constraint) + yield constraint_qos + constraint_schedule = Constraint() + constraint_schedule.action = ConstraintActionEnum.CONSTRAINTACTION_SET + constraint_schedule.schedule.CopyFrom(Constraint_Schedule(start_timestamp=request.start_timestamp, duration_days=request.duration/86400)) + yield constraint_schedule diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 37110c0b6..8d8e455f2 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -14,6 +14,7 @@ import pytest from qos_profile.client.QoSProfileClient import QoSProfileClient +from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile @pytest.fixture(scope='function') def qos_profile_client(): @@ -21,3 +22,26 @@ def qos_profile_client(): yield _client _client.close() + + +def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: + def create_QoSProfileValueUnitPair(data) -> QoSProfileValueUnitPair: + return QoSProfileValueUnitPair(value=data['value'], unit=data['unit']) + qos_profile = QoSProfile() + qos_profile.qos_profile_id.CopyFrom(QoSProfileId(qos_profile_id=Uuid(uuid=qos_profile_data['qos_profile_id']))) + qos_profile.name = qos_profile_data['name'] + qos_profile.description = qos_profile_data['description'] + qos_profile.status = qos_profile_data['status'] + qos_profile.targetMinUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinUpstreamRate'])) + qos_profile.maxUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamRate'])) + qos_profile.maxUpstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamBurstRate'])) + qos_profile.targetMinDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinDownstreamRate'])) + qos_profile.maxDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamRate'])) + qos_profile.maxDownstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamBurstRate'])) + qos_profile.minDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['minDuration'])) + qos_profile.maxDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDuration'])) + qos_profile.priority = qos_profile_data['priority'] + qos_profile.packetDelayBudget.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['packetDelayBudget'])) + qos_profile.jitter.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['jitter'])) + qos_profile.packetErrorLossRate = qos_profile_data['packetErrorLossRate'] + return qos_profile \ No newline at end of file diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py new file mode 100644 index 000000000..51f160fc5 --- /dev/null +++ b/src/qos_profile/tests/test_constraints.py @@ -0,0 +1,142 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from google.protobuf.json_format import MessageToDict + +from common.proto.context_pb2 import QoDConstraintsRequest +from common.tools.grpc.Tools import grpc_message_to_json_string +from qos_profile.client.QoSProfileClient import QoSProfileClient + +from .conftest import create_qos_profile_from_json + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +qos_profile_data = { + "qos_profile_id": "0afc905f-f1f0-4ae2-9925-9df17140b8bf", + "name": "QCI_2_voice", + "description": "QoS profile for game streaming", + "status": "ACTIVE", + "targetMinUpstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxUpstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxUpstreamBurstRate": { + "value": 5, + "unit": "bps" + }, + "targetMinDownstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxDownstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxDownstreamBurstRate": { + "value": 5, + "unit": "bps" + }, + "minDuration": { + "value": 5, + "unit": "Minutes" + }, + "maxDuration": { + "value": 6, + "unit": "Minutes" + }, + "priority": 5, + "packetDelayBudget": { + "value": 5, + "unit": "Minutes" + }, + "jitter": { + "value": 5, + "unit": "Minutes" + }, + "packetErrorLossRate": 3 +} + +target_qos_profile_constraint = { + "action": "CONSTRAINTACTION_SET", + "qos_profile": { + "target_min_upstream_rate": { + "value": 5, + "unit": "bps" + }, + "max_upstream_rate": { + "value": 5, + "unit": "bps" + }, + "max_upstream_burst_rate": { + "value": 5, + "unit": "bps" + }, + "target_min_downstream_rate": { + "value": 5, + "unit": "bps" + }, + "max_downstream_rate": { + "value": 5, + "unit": "bps" + }, + "max_downstream_burst_rate": { + "value": 5, + "unit": "bps" + }, + "min_duration": { + "value": 5, + "unit": "Minutes" + }, + "max_duration": { + "value": 6, + "unit": "Minutes" + }, + "priority": 5, + "packet_delay_budget": { + "value": 5, + "unit": "Minutes" + }, + "jitter": { + "value": 5, + "unit": "Minutes" + }, + "packet_error_loss_rate": 3 + } +} + + +def test_get_constraints(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_created))) + constraints = list(qos_profile_client.GetConstraintListFromQoSProfile(QoDConstraintsRequest( + qos_profile_id=qos_profile.qos_profile_id, start_timestamp=1726063284.25332, duration=86400) + )) + constraint_1 = constraints[0] + constraint_2 = constraints[1] + assert len(constraints) == 2 + assert constraint_1.WhichOneof('constraint') == 'qos_profile' + print(MessageToDict(constraint_1, preserving_proto_field_name=True)) + assert MessageToDict(constraint_1, preserving_proto_field_name=True) == target_qos_profile_constraint + assert constraint_2.WhichOneof('constraint') == 'schedule' + assert constraint_2.schedule.start_timestamp == 1726063284.25332 + assert constraint_2.schedule.duration_days == 1 + + qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) \ No newline at end of file diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index 89bc87322..b98351ce9 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -14,8 +14,8 @@ from grpc import RpcError, StatusCode import logging, pytest -from common.proto.context_pb2 import Empty, Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile - +from .conftest import create_qos_profile_from_json +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient @@ -71,28 +71,6 @@ qos_profile_data = { "packetErrorLossRate": 3 } -def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: - def create_QoSProfileValueUnitPair(data) -> QoSProfileValueUnitPair: - return QoSProfileValueUnitPair(value=data['value'], unit=data['unit']) - qos_profile = QoSProfile() - qos_profile.qos_profile_id.CopyFrom(QoSProfileId(qos_profile_id=Uuid(uuid=qos_profile_data['qos_profile_id']))) - qos_profile.name = qos_profile_data['name'] - qos_profile.description = qos_profile_data['description'] - qos_profile.status = qos_profile_data['status'] - qos_profile.targetMinUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinUpstreamRate'])) - qos_profile.maxUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamRate'])) - qos_profile.maxUpstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamBurstRate'])) - qos_profile.targetMinDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinDownstreamRate'])) - qos_profile.maxDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamRate'])) - qos_profile.maxDownstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamBurstRate'])) - qos_profile.minDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['minDuration'])) - qos_profile.maxDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDuration'])) - qos_profile.priority = qos_profile_data['priority'] - qos_profile.packetDelayBudget.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['packetDelayBudget'])) - qos_profile.jitter.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['jitter'])) - qos_profile.packetErrorLossRate = qos_profile_data['packetErrorLossRate'] - return qos_profile - def test_create_qos_profile(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) -- GitLab From 4788a506c97bb3085278f3bd0fef802e30bef90f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 15:08:57 +0000 Subject: [PATCH 09/16] Updated QoS Profile gRPC port --- manifests/qos_profileservice.yaml | 10 +++++----- src/qos_profile/tests/conftest.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/manifests/qos_profileservice.yaml b/manifests/qos_profileservice.yaml index 1bcaa500f..2ad7fa055 100644 --- a/manifests/qos_profileservice.yaml +++ b/manifests/qos_profileservice.yaml @@ -32,17 +32,17 @@ spec: image: labs.etsi.org:5050/tfs/controller/qos_profile:latest imagePullPolicy: Always ports: - - containerPort: 30060 + - containerPort: 20040 - containerPort: 9192 env: - name: LOG_LEVEL value: "INFO" readinessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30060"] + command: ["/bin/grpc_health_probe", "-addr=:20040"] livenessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30060"] + command: ["/bin/grpc_health_probe", "-addr=:20040"] resources: requests: cpu: 250m @@ -64,8 +64,8 @@ spec: ports: - name: grpc protocol: TCP - port: 30060 - targetPort: 30060 + port: 20040 + targetPort: 20040 - name: metrics protocol: TCP port: 9192 diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 8d8e455f2..5c95fd62c 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -18,7 +18,7 @@ from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId @pytest.fixture(scope='function') def qos_profile_client(): - _client = QoSProfileClient(host='0.0.0.0', port=30060) + _client = QoSProfileClient(host='0.0.0.0', port=20040) yield _client _client.close() -- GitLab From 0c5c6807e06750918309af19ea0889ee26b47007 Mon Sep 17 00:00:00 2001 From: hajipour Date: Thu, 26 Sep 2024 11:52:22 +0200 Subject: [PATCH 10/16] feat: Move QoSProfile OPs form context to qos_profile module. --- deploy/tfs.sh | 11 +++ manifests/qos_profileservice.yaml | 5 +- my_deploy.sh | 3 +- proto/context.proto | 63 ++-------------- proto/qos_profile.proto | 48 ++++++++++-- src/context/client/ContextClient.py | 37 +--------- .../service/ContextServiceServicerImpl.py | 31 +------- src/qos_profile/Dockerfile | 2 +- src/qos_profile/client/QoSProfileClient.py | 3 +- src/qos_profile/requirements.in | 5 ++ src/qos_profile/service/QoSProfileService.py | 5 +- .../service/QoSProfileServiceServicerImpl.py | 68 +++++++---------- src/qos_profile/service/__main__.py | 15 +++- src/qos_profile/service/database/Engine.py | 55 ++++++++++++++ .../service/database/QoSProfile.py | 0 src/qos_profile/service/database/__init__.py | 14 ++++ .../service/database/models/QoSProfile.py | 3 +- .../service/database/models/_Base.py | 74 +++++++++++++++++++ .../service/database/models/__init__.py | 13 ++++ src/qos_profile/tests/conftest.py | 2 + src/qos_profile/tests/test_crud.py | 1 + 21 files changed, 281 insertions(+), 177 deletions(-) create mode 100644 src/qos_profile/service/database/Engine.py rename src/{context => qos_profile}/service/database/QoSProfile.py (100%) create mode 100644 src/qos_profile/service/database/__init__.py rename src/{context => qos_profile}/service/database/models/QoSProfile.py (93%) create mode 100644 src/qos_profile/service/database/models/_Base.py create mode 100644 src/qos_profile/service/database/models/__init__.py diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 62f36a2c1..f72067455 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -159,6 +159,17 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t --from-literal=CRDB_SSLMODE=require printf "\n" +echo "Create secret with CockroachDB data for QoSProfile" +CRDB_DATABASE_QoSProfile="tfs_qos_profile" # TODO: change by specific configurable environment variable +kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ + --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ + --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QoSProfile} \ + --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ + --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ + --from-literal=CRDB_SSLMODE=require +printf "\n" + echo "Create secret with NATS data" NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') if [ -z "$NATS_CLIENT_PORT" ]; then diff --git a/manifests/qos_profileservice.yaml b/manifests/qos_profileservice.yaml index 1bcaa500f..0a27787f3 100644 --- a/manifests/qos_profileservice.yaml +++ b/manifests/qos_profileservice.yaml @@ -37,6 +37,9 @@ spec: env: - name: LOG_LEVEL value: "INFO" + envFrom: + - secretRef: + name: crdb-qos-profile-data readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:30060"] @@ -69,4 +72,4 @@ spec: - name: metrics protocol: TCP port: 9192 - targetPort: 9192 + targetPort: 9192 \ No newline at end of file diff --git a/my_deploy.sh b/my_deploy.sh index b89df7481..922e72ba6 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,7 +20,8 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +# export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" +export TFS_COMPONENTS="context device pathcomp service nbi qos_profile" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" diff --git a/proto/context.proto b/proto/context.proto index 7f99c4525..65a2344c6 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -68,12 +68,6 @@ service ContextService { rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} rpc SelectSlice (SliceFilter ) returns ( SliceList ) {} - rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} - rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} - rpc DeleteQoSProfile (QoSProfileId ) returns ( Empty ) {} - rpc GetQoSProfile (QoSProfileId ) returns ( QoSProfile ) {} - rpc GetQoSProfiles (Empty ) returns (stream QoSProfile ) {} - rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} rpc GetConnection (ConnectionId ) returns ( Connection ) {} @@ -408,43 +402,6 @@ message SliceEvent { SliceId slice_id = 2; } - -// ----- QoSProfile ---------------------------------------------------------------------------------------------------- -message QoSProfileId { - Uuid qos_profile_id = 1; -} - -message QoSProfileValueUnitPair { - int32 value = 1; - string unit = 2; -} - -message QoDConstraintsRequest { - QoSProfileId qos_profile_id = 1; - double start_timestamp = 2; - float duration = 3; -} - -message QoSProfile { - QoSProfileId qos_profile_id = 1; - string name = 2; - string description = 3; - string status = 4; - QoSProfileValueUnitPair targetMinUpstreamRate = 5; - QoSProfileValueUnitPair maxUpstreamRate = 6; - QoSProfileValueUnitPair maxUpstreamBurstRate = 7; - QoSProfileValueUnitPair targetMinDownstreamRate = 8; - QoSProfileValueUnitPair maxDownstreamRate = 9; - QoSProfileValueUnitPair maxDownstreamBurstRate = 10; - QoSProfileValueUnitPair minDuration = 11; - QoSProfileValueUnitPair maxDuration = 12; - int32 priority = 13; - QoSProfileValueUnitPair packetDelayBudget = 14; - QoSProfileValueUnitPair jitter = 15; - int32 packetErrorLossRate = 16; -} - - // ----- Connection ---------------------------------------------------------------------------------------------------- message ConnectionId { Uuid connection_uuid = 1; @@ -640,20 +597,14 @@ message Constraint_Exclusions { repeated LinkId link_ids = 4; } -message Constraint_QoSProfile { - QoSProfileValueUnitPair target_min_upstream_rate = 1; - QoSProfileValueUnitPair max_upstream_rate = 2; - QoSProfileValueUnitPair max_upstream_burst_rate = 3; - QoSProfileValueUnitPair target_min_downstream_rate = 4; - QoSProfileValueUnitPair max_downstream_rate = 5; - QoSProfileValueUnitPair max_downstream_burst_rate = 6; - QoSProfileValueUnitPair min_duration = 7; - QoSProfileValueUnitPair max_duration = 8; - int32 priority = 9; - QoSProfileValueUnitPair packet_delay_budget = 10; - QoSProfileValueUnitPair jitter = 11; - int32 packet_error_loss_rate = 12; +message QoSProfileId { + context.Uuid qos_profile_id = 1; +} + +message Constraint_QoSProfile { + QoSProfileId qos_profile_id = 1; + string qos_profile_name = 2; } message Constraint { diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto index 557cef539..1237314e1 100644 --- a/proto/qos_profile.proto +++ b/proto/qos_profile.proto @@ -17,11 +17,47 @@ package qos_profile; import "context.proto"; + +message QoSProfileId { + context.Uuid qos_profile_id = 1; +} + +message QoSProfileValueUnitPair { + int32 value = 1; + string unit = 2; +} + +message QoDConstraintsRequest { + QoSProfileId qos_profile_id = 1; + double start_timestamp = 2; + float duration = 3; +} + +message QoSProfile { + QoSProfileId qos_profile_id = 1; + string name = 2; + string description = 3; + string status = 4; + QoSProfileValueUnitPair targetMinUpstreamRate = 5; + QoSProfileValueUnitPair maxUpstreamRate = 6; + QoSProfileValueUnitPair maxUpstreamBurstRate = 7; + QoSProfileValueUnitPair targetMinDownstreamRate = 8; + QoSProfileValueUnitPair maxDownstreamRate = 9; + QoSProfileValueUnitPair maxDownstreamBurstRate = 10; + QoSProfileValueUnitPair minDuration = 11; + QoSProfileValueUnitPair maxDuration = 12; + int32 priority = 13; + QoSProfileValueUnitPair packetDelayBudget = 14; + QoSProfileValueUnitPair jitter = 15; + int32 packetErrorLossRate = 16; +} + + service QoSProfileService { - rpc CreateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} - rpc UpdateQoSProfile (context.QoSProfile ) returns (context.QoSProfile ) {} - rpc DeleteQoSProfile (context.QoSProfileId ) returns (context.Empty ) {} - rpc GetQoSProfile (context.QoSProfileId ) returns (context.QoSProfile ) {} - rpc GetQoSProfiles (context.Empty ) returns (stream context.QoSProfile ) {} - rpc GetConstraintListFromQoSProfile (context.QoDConstraintsRequest) returns (stream context.Constraint ) {} + rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {} + rpc DeleteQoSProfile (QoSProfileId ) returns (context.Empty ) {} + rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} + rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint ) {} } diff --git a/src/context/client/ContextClient.py b/src/context/client/ContextClient.py index 024381c54..2776a0d29 100644 --- a/src/context/client/ContextClient.py +++ b/src/context/client/ContextClient.py @@ -27,7 +27,7 @@ from common.proto.context_pb2 import ( Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList, Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList, Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList, - OpticalConfig, OpticalConfigId, OpticalConfigList, QoSProfileId, QoSProfile + OpticalConfig, OpticalConfigId, OpticalConfigList ) from common.proto.context_pb2_grpc import ContextServiceStub from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub @@ -362,41 +362,6 @@ class ContextClient: LOGGER.debug('GetSliceEvents result: {:s}'.format(grpc_message_to_json_string(response))) return response - @RETRY_DECORATOR - def CreateQoSProfile(self, request : QoSProfile) -> QoSProfile: - LOGGER.debug('CreateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.CreateQoSProfile(request) - LOGGER.debug('CreateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) - return response - - @RETRY_DECORATOR - def UpdateQoSProfile(self, request : QoSProfile) -> QoSProfile: - LOGGER.debug('UpdateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.UpdateQoSProfile(request) - LOGGER.debug('UpdateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) - return response - - @RETRY_DECORATOR - def DeleteQoSProfile(self, request : QoSProfileId) -> Empty: - LOGGER.debug('DeleteQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.DeleteQoSProfile(request) - LOGGER.debug('DeleteQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) - return response - - @RETRY_DECORATOR - def GetQoSProfile(self, request : QoSProfileId) -> QoSProfile: - LOGGER.debug('GetQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.GetQoSProfile(request) - LOGGER.debug('GetQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) - return response - - @RETRY_DECORATOR - def GetQoSProfiles(self, request : Empty) -> Iterator[QoSProfile]: - LOGGER.debug('GetQoSProfiles request: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.GetQoSProfiles(request) - LOGGER.debug('GetQoSProfiles result: {:s}'.format(grpc_message_to_json_string(response))) - return response - @RETRY_DECORATOR def ListConnectionIds(self, request: ServiceId) -> ConnectionIdList: LOGGER.debug('ListConnectionIds request: {:s}'.format(grpc_message_to_json_string(request))) diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index e6c305f2f..be3237210 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -24,7 +24,7 @@ from common.proto.context_pb2 import ( Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList, Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList, Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList, - OpticalConfigList, OpticalConfigId, OpticalConfig, QoSProfileId, QoSProfile + OpticalConfigList, OpticalConfigId, OpticalConfig ) from common.proto.policy_pb2 import PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule from common.proto.context_pb2_grpc import ContextServiceServicer @@ -46,7 +46,6 @@ from .database.Slice import ( from .database.Topology import ( topology_delete, topology_get, topology_get_details, topology_list_ids, topology_list_objs, topology_set) from .database.OpticalConfig import set_opticalconfig, select_opticalconfig, get_opticalconfig -from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles LOGGER = logging.getLogger(__name__) @@ -252,34 +251,6 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer for message in consume_events(self.messagebroker, {EventTopicEnum.SLICE}): yield message - # ----- QoSProfile ----------------------------------------------------------------------------------------------- - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def CreateQoSProfile(self, request : QoSProfile, context : grpc.ServicerContext) -> QoSProfile: - return set_qos_profile(self.db_engine, request) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def UpdateQoSProfile(self, request : QoSProfile, context : grpc.ServicerContext) -> QoSProfile: - return set_qos_profile(self.db_engine, request) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> Empty: - return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetQoSProfile(self, request : QoSProfileId, context : grpc.ServicerContext) -> QoSProfile: - qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) - if qos_profile is None: - context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - return QoSProfile() - return qos_profile - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetQoSProfiles(self, request : Empty, context : grpc.ServicerContext) -> Iterator[QoSProfile]: - yield from get_qos_profiles(self.db_engine, request) - - # ----- Connection ------------------------------------------------------------------------------------------------- @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile index a35c2e741..d8576ac49 100644 --- a/src/qos_profile/Dockerfile +++ b/src/qos_profile/Dockerfile @@ -56,7 +56,7 @@ RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/qos_profile WORKDIR /var/teraflow/qos_profile -COPY src/service/requirements.in requirements.in +COPY src/qos_profile/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py index e7bec8739..d243323a1 100644 --- a/src/qos_profile/client/QoSProfileClient.py +++ b/src/qos_profile/client/QoSProfileClient.py @@ -16,7 +16,8 @@ from typing import Iterator import grpc, logging from common.Constants import ServiceNameEnum from common.Settings import get_service_host, get_service_port_grpc -from common.proto.context_pb2 import Empty, QoSProfileId, QoSProfile, QoDConstraintsRequest, Constraint +from common.proto.context_pb2 import Empty +from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string diff --git a/src/qos_profile/requirements.in b/src/qos_profile/requirements.in index 5cf553eaa..9ea7059c4 100644 --- a/src/qos_profile/requirements.in +++ b/src/qos_profile/requirements.in @@ -13,3 +13,8 @@ # limitations under the License. + +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* \ No newline at end of file diff --git a/src/qos_profile/service/QoSProfileService.py b/src/qos_profile/service/QoSProfileService.py index bdc90f5bc..ce5c5591b 100644 --- a/src/qos_profile/service/QoSProfileService.py +++ b/src/qos_profile/service/QoSProfileService.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sqlalchemy from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc from common.proto.qos_profile_pb2_grpc import add_QoSProfileServiceServicer_to_server @@ -19,10 +20,10 @@ from common.tools.service.GenericGrpcService import GenericGrpcService from .QoSProfileServiceServicerImpl import QoSProfileServiceServicerImpl class QoSProfileService(GenericGrpcService): - def __init__(self, cls_name: str = __name__) -> None: + def __init__(self, db_engine: sqlalchemy.engine.Engine, cls_name: str = __name__) -> None: port = get_service_port_grpc(ServiceNameEnum.QOSPROFILE) super().__init__(port, cls_name=cls_name) - self.qos_profile_servicer = QoSProfileServiceServicerImpl() + self.qos_profile_servicer = QoSProfileServiceServicerImpl(db_engine) def install_servicers(self): add_QoSProfileServiceServicer_to_server(self.qos_profile_servicer, self.server) diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index bdcc3e8c3..860ee2ee3 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc, logging +import grpc, logging, sqlalchemy from typing import Iterator import grpc._channel from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import QoDConstraintsRequest, Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId, QoSProfile +from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty +from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer from context.client.ContextClient import ContextClient +from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles LOGGER = logging.getLogger(__name__) @@ -27,65 +29,49 @@ LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('QoSProfile', 'RPC') class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): - def __init__(self ) -> None: + def __init__(self, db_engine: sqlalchemy.engine.Engine) -> None: LOGGER.debug('Servicer Created') + self.db_engine = db_engine @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - context_client = ContextClient() - try: - qos_profile_get = context_client.GetQoSProfile(request.qos_profile_id) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is not None: context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists') context.set_code(grpc.StatusCode.ALREADY_EXISTS) return QoSProfile() - except grpc._channel._InactiveRpcError as exc: - if exc.code() != grpc.StatusCode.NOT_FOUND: - raise exc - qos_profile = context_client.CreateQoSProfile(request) - return qos_profile + return set_qos_profile(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - context_client = ContextClient() - try: - _ = context_client.GetQoSProfile(request.qos_profile_id) - except grpc._channel._InactiveRpcError as exc: - if exc.code() == grpc.StatusCode.NOT_FOUND: - context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - return QoSProfile() - qos_profile = context_client.UpdateQoSProfile(request) - return qos_profile + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return set_qos_profile(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty: - context_client = ContextClient() - try: - _ = context_client.GetQoSProfile(request) - except grpc._channel._InactiveRpcError as exc: - if exc.code() == grpc.StatusCode.NOT_FOUND: - context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - return QoSProfile() - empty = context_client.DeleteQoSProfile(request) - return empty + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> QoSProfile: - context_client = ContextClient() - try: - qos_profile = context_client.GetQoSProfile(request) - except grpc._channel._InactiveRpcError as exc: - if exc.code() == grpc.StatusCode.NOT_FOUND: - context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - return QoSProfile() + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() return qos_profile @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetQoSProfiles(self, request: Empty, context: grpc.ServicerContext) -> Iterator[QoSProfile]: - context_client = ContextClient() - yield from context_client.GetQoSProfiles(request) + yield from get_qos_profiles(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py index ccd1ca23f..6666d9f19 100644 --- a/src/qos_profile/service/__main__.py +++ b/src/qos_profile/service/__main__.py @@ -19,6 +19,7 @@ from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, wait_for_environment_variables ) +from .database.Engine import Engine from .QoSProfileService import QoSProfileService terminate = threading.Event() @@ -49,8 +50,20 @@ def main(): metrics_port = get_metrics_port() start_http_server(metrics_port) + # Get Database Engine instance and initialize database, if needed + LOGGER.info('Getting SQLAlchemy DB Engine...') + db_engine = Engine.get_engine() + if db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return -1 + + try: + Engine.create_database(db_engine) + except: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_engine.url))) + # Starting service service - grpc_service = QoSProfileService() + grpc_service = QoSProfileService(db_engine) grpc_service.start() # Wait for Ctrl+C or termination signal diff --git a/src/qos_profile/service/database/Engine.py b/src/qos_profile/service/database/Engine.py new file mode 100644 index 000000000..6ba1a82d0 --- /dev/null +++ b/src/qos_profile/service/database/Engine.py @@ -0,0 +1,55 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sqlalchemy, sqlalchemy_utils +from common.Settings import get_setting + +LOGGER = logging.getLogger(__name__) + +APP_NAME = 'tfs' +ECHO = False # true: dump SQL commands and transactions executed +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@qos-profileservice.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' + +class Engine: + @staticmethod + def get_engine() -> sqlalchemy.engine.Engine: + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = get_setting('CRDB_DATABASE') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + + try: + engine = sqlalchemy.create_engine( + crdb_uri, connect_args={'application_name': APP_NAME}, echo=ECHO, future=True) + except: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) + return None + + return engine + + @staticmethod + def create_database(engine : sqlalchemy.engine.Engine) -> None: + if not sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.create_database(engine.url) + + @staticmethod + def drop_database(engine : sqlalchemy.engine.Engine) -> None: + if sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.drop_database(engine.url) diff --git a/src/context/service/database/QoSProfile.py b/src/qos_profile/service/database/QoSProfile.py similarity index 100% rename from src/context/service/database/QoSProfile.py rename to src/qos_profile/service/database/QoSProfile.py diff --git a/src/qos_profile/service/database/__init__.py b/src/qos_profile/service/database/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/qos_profile/service/database/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/context/service/database/models/QoSProfile.py b/src/qos_profile/service/database/models/QoSProfile.py similarity index 93% rename from src/context/service/database/models/QoSProfile.py rename to src/qos_profile/service/database/models/QoSProfile.py index 431d0f503..bfbdeef0a 100644 --- a/src/context/service/database/models/QoSProfile.py +++ b/src/qos_profile/service/database/models/QoSProfile.py @@ -13,13 +13,14 @@ # limitations under the License. from sqlalchemy import Column, Integer, String, JSON +from sqlalchemy.dialects.postgresql import UUID from ._Base import _Base class QoSProfileModel(_Base): __tablename__ = 'qos_profile' - qos_profile_id = Column(String, primary_key=True) + qos_profile_id = Column(UUID(as_uuid=False), primary_key=True) name = Column(String, nullable=False) description = Column(String, nullable=False) status = Column(String, nullable=False) diff --git a/src/qos_profile/service/database/models/_Base.py b/src/qos_profile/service/database/models/_Base.py new file mode 100644 index 000000000..6e71b3c0f --- /dev/null +++ b/src/qos_profile/service/database/models/_Base.py @@ -0,0 +1,74 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlalchemy +from typing import Any, List +from sqlalchemy.orm import Session, sessionmaker, declarative_base +from sqlalchemy.sql import text +from sqlalchemy_cockroachdb import run_transaction + +_Base = declarative_base() + +def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: + def index_storing( + index_name : str, table_name : str, index_fields : List[str], storing_fields : List[str] + ) -> Any: + str_index_fields = ','.join(['"{:s}"'.format(index_field) for index_field in index_fields]) + str_storing_fields = ','.join(['"{:s}"'.format(storing_field) for storing_field in storing_fields]) + INDEX_STORING = 'CREATE INDEX IF NOT EXISTS {:s} ON "{:s}" ({:s}) STORING ({:s});' + return text(INDEX_STORING.format(index_name, table_name, str_index_fields, str_storing_fields)) + + statements = [ + index_storing('device_configrule_device_uuid_rec_idx', 'device_configrule', ['device_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + ]), + index_storing('service_configrule_service_uuid_rec_idx', 'service_configrule', ['service_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + ]), + index_storing('slice_configrule_slice_uuid_rec_idx', 'slice_configrule', ['slice_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + ]), + index_storing('connection_service_uuid_rec_idx', 'connection', ['service_uuid'], [ + 'settings', 'created_at', 'updated_at' + ]), + index_storing('service_constraint_service_uuid_rec_idx', 'service_constraint', ['service_uuid'], [ + 'position', 'kind', 'data', 'created_at', 'updated_at' + ]), + index_storing('slice_constraint_slice_uuid_rec_idx', 'slice_constraint', ['slice_uuid'], [ + 'position', 'kind', 'data', 'created_at', 'updated_at' + ]), + index_storing('endpoint_device_uuid_rec_idx', 'endpoint', ['device_uuid'], [ + 'topology_uuid', 'name', 'endpoint_type', 'kpi_sample_types', 'created_at', 'updated_at' + ]), + index_storing('qos_profile_context_uuid_rec_idx', 'qos_profile', ['context_uuid'], [ + 'service_name', 'service_type', 'service_status', 'created_at', 'updated_at' + ]), + index_storing('slice_context_uuid_rec_idx', 'slice', ['context_uuid'], [ + 'slice_name', 'slice_status', 'slice_owner_uuid', 'slice_owner_string', 'created_at', 'updated_at' + ]), + index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [ + 'topology_name', 'created_at', 'updated_at' + ]), + index_storing('device_component_idx', 'device_component', ['device_uuid'], [ + 'name', 'type', 'attributes', 'created_at', 'updated_at' + ]), + ] + def callback(session : Session) -> bool: + for stmt in statements: session.execute(stmt) + run_transaction(sessionmaker(bind=db_engine), callback) + +def rebuild_database(db_engine : sqlalchemy.engine.Engine, drop_if_exists : bool = False): + if drop_if_exists: _Base.metadata.drop_all(db_engine) + _Base.metadata.create_all(db_engine) + # create_performance_enhancers(db_engine) diff --git a/src/qos_profile/service/database/models/__init__.py b/src/qos_profile/service/database/models/__init__.py new file mode 100644 index 000000000..bbfc943b6 --- /dev/null +++ b/src/qos_profile/service/database/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 8d8e455f2..e0ae3490c 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -15,6 +15,8 @@ import pytest from qos_profile.client.QoSProfileClient import QoSProfileClient from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile +from common.proto.context_pb2 import Uuid +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfileId, QoSProfile @pytest.fixture(scope='function') def qos_profile_client(): diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index b98351ce9..e60f34933 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -16,6 +16,7 @@ from grpc import RpcError, StatusCode import logging, pytest from .conftest import create_qos_profile_from_json from common.proto.context_pb2 import Empty, Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileId from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient -- GitLab From 9b7ed9ee82b7dfc2cb8505839b32d030f24e2adb Mon Sep 17 00:00:00 2001 From: hajipour Date: Fri, 27 Sep 2024 11:24:22 +0200 Subject: [PATCH 11/16] feat: QoSProfile database moved to logical database in CRDB and context interaction removed from QoSProfile component: - tests updated - qos_profile constraint aded to ConstraintKindEnum and Constraint parser of context database - GenericDatabase class used to create qos_profile database - qos_profile component activation by default added to my_deploy.sh --- .gitignore | 1 + deploy/crdb.sh | 3 ++ deploy/tfs.sh | 3 +- my_deploy.sh | 6 ++- proto/qos_profile.proto | 19 +++---- src/context/service/database/Constraint.py | 3 +- .../database/models/ConstraintModel.py | 1 + src/qos_profile/Dockerfile | 8 --- src/qos_profile/client/QoSProfileClient.py | 5 +- .../service/QoSProfileServiceServicerImpl.py | 38 +++++-------- src/qos_profile/service/__main__.py | 21 ++++---- .../service/database/QoSProfile.py | 4 +- src/qos_profile/tests/conftest.py | 5 +- src/qos_profile/tests/test_constraints.py | 54 ++----------------- src/qos_profile/tests/test_crud.py | 5 +- 15 files changed, 54 insertions(+), 122 deletions(-) diff --git a/.gitignore b/.gitignore index e1f87cfd3..a0ac78095 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.my_venv/ # requirements.txt # removed to enable tracking versions of packages over time # PyInstaller diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 3e80b6350..474c32ef7 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -171,6 +171,9 @@ function crdb_drop_database_single() { kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" + kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ + ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ + --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE_QOSPROFILE};" echo } diff --git a/deploy/tfs.sh b/deploy/tfs.sh index c4f5b106f..e3a1a03d9 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -202,11 +202,10 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty printf "\n" echo "Create secret with CockroachDB data for QoSProfile" -CRDB_DATABASE_QoSProfile="tfs_qos_profile" # TODO: change by specific configurable environment variable kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ - --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QoSProfile} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QOSPROFILE} \ --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ --from-literal=CRDB_SSLMODE=require diff --git a/my_deploy.sh b/my_deploy.sh index f0f4e8f6b..60084cc76 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,8 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -# export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" -export TFS_COMPONENTS="context device pathcomp service nbi qos_profile" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -117,6 +116,9 @@ export CRDB_PASSWORD="tfs123" # Set the database name to be used by Context. export CRDB_DATABASE="tfs" +# Set the database to be used by the QoSProfile component +export CRDB_DATABASE_QOSPROFILE="tfs_qos_profile" + # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto index 1237314e1..dc8348766 100644 --- a/proto/qos_profile.proto +++ b/proto/qos_profile.proto @@ -17,24 +17,19 @@ package qos_profile; import "context.proto"; - -message QoSProfileId { - context.Uuid qos_profile_id = 1; -} - message QoSProfileValueUnitPair { int32 value = 1; string unit = 2; } message QoDConstraintsRequest { - QoSProfileId qos_profile_id = 1; + context.QoSProfileId qos_profile_id = 1; double start_timestamp = 2; float duration = 3; } message QoSProfile { - QoSProfileId qos_profile_id = 1; + context.QoSProfileId qos_profile_id = 1; string name = 2; string description = 3; string status = 4; @@ -54,10 +49,10 @@ message QoSProfile { service QoSProfileService { - rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc DeleteQoSProfile (QoSProfileId ) returns (context.Empty ) {} - rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {} - rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} + rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint ) {} } diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index db96ed9de..0b0422192 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -69,7 +69,8 @@ def compose_constraints_data( constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid) elif kind in { ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY, - ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS + ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS, + ConstraintKindEnum.QOS_PROFILE }: constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value) else: diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py index fc56a1145..3eef030fc 100644 --- a/src/context/service/database/models/ConstraintModel.py +++ b/src/context/service/database/models/ConstraintModel.py @@ -31,6 +31,7 @@ class ConstraintKindEnum(enum.Enum): SLA_LATENCY = 'sla_latency' SLA_AVAILABILITY = 'sla_availability' SLA_ISOLATION = 'sla_isolation' + QOS_PROFILE = 'qos_profile' EXCLUSIONS = 'exclusions' class ServiceConstraintModel(_Base): diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile index d8576ac49..361dc588c 100644 --- a/src/qos_profile/Dockerfile +++ b/src/qos_profile/Dockerfile @@ -62,14 +62,6 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/__init__.py context/__init__.py -COPY src/context/client/. context/client/ -COPY src/device/__init__.py device/__init__.py -COPY src/device/client/. device/client/ -COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py -COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/ -COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py -COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ COPY src/qos_profile/. qos_profile/ # Start the service diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py index d243323a1..c6ca46dce 100644 --- a/src/qos_profile/client/QoSProfileClient.py +++ b/src/qos_profile/client/QoSProfileClient.py @@ -16,8 +16,9 @@ from typing import Iterator import grpc, logging from common.Constants import ServiceNameEnum from common.Settings import get_service_host, get_service_port_grpc -from common.proto.context_pb2 import Empty -from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest +from common.proto.context_pb2 import Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest +from common.proto.context_pb2 import Constraint from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index 860ee2ee3..47f5fbb25 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -17,10 +17,9 @@ from typing import Iterator import grpc._channel from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty -from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest +from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer -from context.client.ContextClient import ContextClient from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles @@ -35,7 +34,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) if qos_profile is not None: context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists') context.set_code(grpc.StatusCode.ALREADY_EXISTS) @@ -44,7 +43,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) if qos_profile is None: context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') context.set_code(grpc.StatusCode.NOT_FOUND) @@ -55,7 +54,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty: qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) if qos_profile is None: - context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') context.set_code(grpc.StatusCode.NOT_FOUND) return QoSProfile() return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) @@ -76,28 +75,15 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]: - context_client = ContextClient() - try: - qos_profile = context_client.GetQoSProfile(request.qos_profile_id) - except grpc._channel._InactiveRpcError as exc: - if exc.code() == grpc.StatusCode.NOT_FOUND: - context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - yield Constraint() + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + yield Constraint() qos_profile_constraint = Constraint_QoSProfile() - qos_profile_constraint.target_min_upstream_rate.CopyFrom(qos_profile.targetMinUpstreamRate) - qos_profile_constraint.max_upstream_rate.CopyFrom(qos_profile.maxUpstreamRate) - qos_profile_constraint.max_upstream_burst_rate.CopyFrom(qos_profile.maxUpstreamBurstRate) - qos_profile_constraint.target_min_downstream_rate.CopyFrom(qos_profile.targetMinDownstreamRate) - qos_profile_constraint.max_downstream_rate.CopyFrom(qos_profile.maxDownstreamRate) - qos_profile_constraint.max_downstream_burst_rate.CopyFrom(qos_profile.maxDownstreamBurstRate) - qos_profile_constraint.min_duration.CopyFrom(qos_profile.minDuration) - qos_profile_constraint.max_duration.CopyFrom(qos_profile.maxDuration) - qos_profile_constraint.priority = qos_profile.priority - qos_profile_constraint.packet_delay_budget.CopyFrom(qos_profile.packetDelayBudget) - qos_profile_constraint.jitter.CopyFrom(qos_profile.jitter) - qos_profile_constraint.packet_error_loss_rate =qos_profile.packetErrorLossRate + qos_profile_constraint.qos_profile_name = qos_profile.name + qos_profile_constraint.qos_profile_id.CopyFrom(qos_profile.qos_profile_id) constraint_qos = Constraint() constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET constraint_qos.qos_profile.CopyFrom(qos_profile_constraint) diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py index 6666d9f19..d734d5567 100644 --- a/src/qos_profile/service/__main__.py +++ b/src/qos_profile/service/__main__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import logging, signal, sys, threading from prometheus_client import start_http_server from common.Constants import ServiceNameEnum @@ -19,7 +20,8 @@ from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, wait_for_environment_variables ) -from .database.Engine import Engine +from common.tools.database.GenericDatabase import Database +from qos_profile.service.database.models.QoSProfile import QoSProfileModel from .QoSProfileService import QoSProfileService terminate = threading.Event() @@ -50,20 +52,17 @@ def main(): metrics_port = get_metrics_port() start_http_server(metrics_port) - # Get Database Engine instance and initialize database, if needed - LOGGER.info('Getting SQLAlchemy DB Engine...') - db_engine = Engine.get_engine() - if db_engine is None: - LOGGER.error('Unable to get SQLAlchemy DB Engine...') - return -1 + db_manager = Database(db_name=os.getenv('CRDB_DATABASE'), model=QoSProfileModel) try: - Engine.create_database(db_engine) - except: # pylint: disable=bare-except # pragma: no cover - LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_engine.url))) + db_manager.create_database() + db_manager.create_tables() + except Exception as e: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url))) + raise e # Starting service service - grpc_service = QoSProfileService(db_engine) + grpc_service = QoSProfileService(db_manager.db_engine) grpc_service.start() # Wait for Ctrl+C or termination signal diff --git a/src/qos_profile/service/database/QoSProfile.py b/src/qos_profile/service/database/QoSProfile.py index 33f6cc6ec..86823c165 100644 --- a/src/qos_profile/service/database/QoSProfile.py +++ b/src/qos_profile/service/database/QoSProfile.py @@ -19,8 +19,8 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_cockroachdb import run_transaction from typing import List, Optional -from common.proto.context_pb2 import Empty, Uuid, QoSProfileId, QoSProfileValueUnitPair, QoSProfile -from common.method_wrappers.ServiceExceptions import NotFoundException +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile from common.tools.grpc.Tools import grpc_message_to_json from .models.QoSProfile import QoSProfileModel diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 1f823253f..7c8424e00 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -14,9 +14,8 @@ import pytest from qos_profile.client.QoSProfileClient import QoSProfileClient -from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile -from common.proto.context_pb2 import Uuid -from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfileId, QoSProfile +from common.proto.context_pb2 import Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile @pytest.fixture(scope='function') def qos_profile_client(): diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py index 51f160fc5..dd03ff7ae 100644 --- a/src/qos_profile/tests/test_constraints.py +++ b/src/qos_profile/tests/test_constraints.py @@ -15,7 +15,7 @@ import logging from google.protobuf.json_format import MessageToDict -from common.proto.context_pb2 import QoDConstraintsRequest +from common.proto.qos_profile_pb2 import QoDConstraintsRequest from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient @@ -73,54 +73,6 @@ qos_profile_data = { "packetErrorLossRate": 3 } -target_qos_profile_constraint = { - "action": "CONSTRAINTACTION_SET", - "qos_profile": { - "target_min_upstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_upstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_upstream_burst_rate": { - "value": 5, - "unit": "bps" - }, - "target_min_downstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_downstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_downstream_burst_rate": { - "value": 5, - "unit": "bps" - }, - "min_duration": { - "value": 5, - "unit": "Minutes" - }, - "max_duration": { - "value": 6, - "unit": "Minutes" - }, - "priority": 5, - "packet_delay_budget": { - "value": 5, - "unit": "Minutes" - }, - "jitter": { - "value": 5, - "unit": "Minutes" - }, - "packet_error_loss_rate": 3 - } -} - def test_get_constraints(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) @@ -133,8 +85,8 @@ def test_get_constraints(qos_profile_client: QoSProfileClient): constraint_2 = constraints[1] assert len(constraints) == 2 assert constraint_1.WhichOneof('constraint') == 'qos_profile' - print(MessageToDict(constraint_1, preserving_proto_field_name=True)) - assert MessageToDict(constraint_1, preserving_proto_field_name=True) == target_qos_profile_constraint + assert constraint_1.qos_profile.qos_profile_id == qos_profile.qos_profile_id + assert constraint_1.qos_profile.qos_profile_name == 'QCI_2_voice' assert constraint_2.WhichOneof('constraint') == 'schedule' assert constraint_2.schedule.start_timestamp == 1726063284.25332 assert constraint_2.schedule.duration_days == 1 diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index e60f34933..1037f3849 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -16,7 +16,6 @@ from grpc import RpcError, StatusCode import logging, pytest from .conftest import create_qos_profile_from_json from common.proto.context_pb2 import Empty, Uuid, QoSProfileId -from common.proto.qos_profile_pb2 import QoSProfileId from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient @@ -94,8 +93,10 @@ def test_get_qos_profile(qos_profile_client: QoSProfileClient): def test_get_qos_profiles(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty())) + the_qos_profile = [q for q in qos_profiles_got if q.qos_profile_id == qos_profile.qos_profile_id] LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got))) - assert qos_profile == qos_profiles_got[0] + assert len(the_qos_profile) == 1 + assert qos_profile == the_qos_profile[0] def test_update_qos_profile(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) -- GitLab From cc43841633c758817c223df080678f4cf8d88b70 Mon Sep 17 00:00:00 2001 From: hajipour Date: Fri, 27 Sep 2024 11:40:45 +0200 Subject: [PATCH 12/16] refactor: context container removed from .gitlab-ci.yml of qos-profile --- src/qos_profile/.gitlab-ci.yml | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml index 29ac9f3b4..f46d63856 100644 --- a/src/qos_profile/.gitlab-ci.yml +++ b/src/qos_profile/.gitlab-ci.yml @@ -50,24 +50,18 @@ unit_test qos_profile: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - # Context-related + # QoSProfile-related - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - - # Context-related - - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - - # QoSProfile-related - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" - - docker pull "$CI_REGISTRY_IMAGE/context:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - # Context preparation + # environment preparation - docker volume create crdb - > docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 @@ -88,19 +82,12 @@ unit_test qos_profile: - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS - > - docker run --name context -d -p 1010:1010 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CONTEXTSERVICE_SERVICE_HOST - # QoSProfile preparation - > docker run --name $IMAGE_NAME -d -p 3030:3030 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "MB_BACKEND=nats" + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG @@ -108,7 +95,6 @@ unit_test qos_profile: # Check status before the tests - sleep 5 - docker ps -a - - docker logs context - docker logs $IMAGE_NAME # Run the tests @@ -121,11 +107,9 @@ unit_test qos_profile: after_script: # Check status after the tests - docker ps -a - - docker logs context - docker logs $IMAGE_NAME - docker rm -f $IMAGE_NAME - - docker rm -f context - docker rm -f $IMAGE_NAME crdb nats - docker volume rm -f crdb -- GitLab From 4d9bce46b2e7519eaecdbc4c9492ba3c321db32c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 2 Oct 2024 12:06:02 +0000 Subject: [PATCH 13/16] Pre-merge code cleanup --- deploy/tfs.sh | 9 --------- manifests/qos_profileservice.yaml | 30 ++++++++++++++++++++++++++++-- proto/qos_profile.proto | 12 ++++++------ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 38acd8b75..ea9f9091d 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -163,15 +163,6 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty --from-literal=KFK_SERVER_PORT=${KFK_SERVER_PORT} printf "\n" -echo "Create secret with CockroachDB data for QoSProfile" -kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ - --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ - --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ - --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QOSPROFILE} \ - --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ - --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ - --from-literal=CRDB_SSLMODE=require -printf "\n" echo "Create secret with NATS data" NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') diff --git a/manifests/qos_profileservice.yaml b/manifests/qos_profileservice.yaml index f6838b717..801607880 100644 --- a/manifests/qos_profileservice.yaml +++ b/manifests/qos_profileservice.yaml @@ -23,6 +23,8 @@ spec: #replicas: 1 template: metadata: + annotations: + config.linkerd.io/skip-outbound-ports: "4222" labels: app: qos-profileservice spec: @@ -37,9 +39,11 @@ spec: env: - name: LOG_LEVEL value: "INFO" + - name: CRDB_DATABASE + value: "tfs_qos_profile" envFrom: - secretRef: - name: crdb-qos-profile-data + name: crdb-data readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:20040"] @@ -72,4 +76,26 @@ spec: - name: metrics protocol: TCP port: 9192 - targetPort: 9192 \ No newline at end of file + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: qos-profileservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: qos-profileservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto index dc8348766..d032addf4 100644 --- a/proto/qos_profile.proto +++ b/proto/qos_profile.proto @@ -49,10 +49,10 @@ message QoSProfile { service QoSProfileService { - rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} - rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} - rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {} - rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {} - rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} - rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint ) {} + rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} + rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint) {} } -- GitLab From d1c351913653883a834ee766d1da8246e22eca6e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 2 Oct 2024 12:06:14 +0000 Subject: [PATCH 14/16] Pre-merge code cleanup --- my_deploy.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index e14f4f540..3c590b9dc 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -123,9 +123,6 @@ export CRDB_PASSWORD="tfs123" # Set the database name to be used by Context. export CRDB_DATABASE="tfs" -# Set the database to be used by the QoSProfile component -export CRDB_DATABASE_QOSPROFILE="tfs_qos_profile" - # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" -- GitLab From 53ba0ee6de7ca289ac87d848c652182dbae07fe6 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 12:22:24 +0000 Subject: [PATCH 15/16] Pre-merge code cleanup --- deploy/crdb.sh | 3 -- deploy/tfs.sh | 1 - my_deploy.sh | 3 ++ src/qos_profile/client/QoSProfileClient.py | 2 +- src/qos_profile/requirements.in | 4 +- src/qos_profile/service/__main__.py | 36 +++++-------- .../service/database/models/_Base.py | 54 +------------------ 7 files changed, 18 insertions(+), 85 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 474c32ef7..3e80b6350 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -171,9 +171,6 @@ function crdb_drop_database_single() { kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" - kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ - ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ - --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE_QOSPROFILE};" echo } diff --git a/deploy/tfs.sh b/deploy/tfs.sh index ea9f9091d..65c1e8de2 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -163,7 +163,6 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty --from-literal=KFK_SERVER_PORT=${KFK_SERVER_PORT} printf "\n" - echo "Create secret with NATS data" NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') if [ -z "$NATS_CLIENT_PORT" ]; then diff --git a/my_deploy.sh b/my_deploy.sh index ad9a2f143..6d0a488c2 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -28,6 +28,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate Monitoring Framework (new) #export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" +# Uncomment to activate QoS Profiles +#export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" + # Uncomment to activate BGP-LS Speaker #export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py index c6ca46dce..748b3f208 100644 --- a/src/qos_profile/client/QoSProfileClient.py +++ b/src/qos_profile/client/QoSProfileClient.py @@ -88,4 +88,4 @@ class QoSProfileClient: LOGGER.debug('GetConstraintListFromQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.GetConstraintListFromQoSProfile(request) LOGGER.debug('GetConstraintListFromQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) - return response \ No newline at end of file + return response diff --git a/src/qos_profile/requirements.in b/src/qos_profile/requirements.in index 9ea7059c4..3e98fef36 100644 --- a/src/qos_profile/requirements.in +++ b/src/qos_profile/requirements.in @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - psycopg2-binary==2.9.* SQLAlchemy==1.4.* sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* \ No newline at end of file +SQLAlchemy-Utils==0.38.* diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py index d734d5567..7f9e6de92 100644 --- a/src/qos_profile/service/__main__.py +++ b/src/qos_profile/service/__main__.py @@ -12,47 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging, signal, sys, threading from prometheus_client import start_http_server -from common.Constants import ServiceNameEnum -from common.Settings import ( - ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, - wait_for_environment_variables -) +from common.Settings import get_log_level, get_metrics_port from common.tools.database.GenericDatabase import Database -from qos_profile.service.database.models.QoSProfile import QoSProfileModel from .QoSProfileService import QoSProfileService +from .database.models.QoSProfile import QoSProfileModel + +LOG_LEVEL = get_log_level() +logging.basicConfig(level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") +LOGGER = logging.getLogger(__name__) + terminate = threading.Event() -LOGGER : logging.Logger = None -def signal_handler(signal, frame): # pylint: disable=redefined-outer-name +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name,unused-argument LOGGER.warning('Terminate signal received') terminate.set() def main(): - global LOGGER # pylint: disable=global-statement - - log_level = get_log_level() - logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") - LOGGER = logging.getLogger(__name__) - - wait_for_environment_variables([ - get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), - get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), - ]) - + LOGGER.info('Starting...') signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.info('Starting...') - # Start metrics server metrics_port = get_metrics_port() start_http_server(metrics_port) - db_manager = Database(db_name=os.getenv('CRDB_DATABASE'), model=QoSProfileModel) + # Get Database Engine instance and initialize database, if needed + db_manager = Database(QoSProfileModel) try: db_manager.create_database() @@ -61,7 +49,7 @@ def main(): LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url))) raise e - # Starting service service + # Starting service grpc_service = QoSProfileService(db_manager.db_engine) grpc_service.start() diff --git a/src/qos_profile/service/database/models/_Base.py b/src/qos_profile/service/database/models/_Base.py index 6e71b3c0f..d94dad3cd 100644 --- a/src/qos_profile/service/database/models/_Base.py +++ b/src/qos_profile/service/database/models/_Base.py @@ -13,62 +13,10 @@ # limitations under the License. import sqlalchemy -from typing import Any, List -from sqlalchemy.orm import Session, sessionmaker, declarative_base -from sqlalchemy.sql import text -from sqlalchemy_cockroachdb import run_transaction +from sqlalchemy.orm import declarative_base _Base = declarative_base() -def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: - def index_storing( - index_name : str, table_name : str, index_fields : List[str], storing_fields : List[str] - ) -> Any: - str_index_fields = ','.join(['"{:s}"'.format(index_field) for index_field in index_fields]) - str_storing_fields = ','.join(['"{:s}"'.format(storing_field) for storing_field in storing_fields]) - INDEX_STORING = 'CREATE INDEX IF NOT EXISTS {:s} ON "{:s}" ({:s}) STORING ({:s});' - return text(INDEX_STORING.format(index_name, table_name, str_index_fields, str_storing_fields)) - - statements = [ - index_storing('device_configrule_device_uuid_rec_idx', 'device_configrule', ['device_uuid'], [ - 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' - ]), - index_storing('service_configrule_service_uuid_rec_idx', 'service_configrule', ['service_uuid'], [ - 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' - ]), - index_storing('slice_configrule_slice_uuid_rec_idx', 'slice_configrule', ['slice_uuid'], [ - 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' - ]), - index_storing('connection_service_uuid_rec_idx', 'connection', ['service_uuid'], [ - 'settings', 'created_at', 'updated_at' - ]), - index_storing('service_constraint_service_uuid_rec_idx', 'service_constraint', ['service_uuid'], [ - 'position', 'kind', 'data', 'created_at', 'updated_at' - ]), - index_storing('slice_constraint_slice_uuid_rec_idx', 'slice_constraint', ['slice_uuid'], [ - 'position', 'kind', 'data', 'created_at', 'updated_at' - ]), - index_storing('endpoint_device_uuid_rec_idx', 'endpoint', ['device_uuid'], [ - 'topology_uuid', 'name', 'endpoint_type', 'kpi_sample_types', 'created_at', 'updated_at' - ]), - index_storing('qos_profile_context_uuid_rec_idx', 'qos_profile', ['context_uuid'], [ - 'service_name', 'service_type', 'service_status', 'created_at', 'updated_at' - ]), - index_storing('slice_context_uuid_rec_idx', 'slice', ['context_uuid'], [ - 'slice_name', 'slice_status', 'slice_owner_uuid', 'slice_owner_string', 'created_at', 'updated_at' - ]), - index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [ - 'topology_name', 'created_at', 'updated_at' - ]), - index_storing('device_component_idx', 'device_component', ['device_uuid'], [ - 'name', 'type', 'attributes', 'created_at', 'updated_at' - ]), - ] - def callback(session : Session) -> bool: - for stmt in statements: session.execute(stmt) - run_transaction(sessionmaker(bind=db_engine), callback) - def rebuild_database(db_engine : sqlalchemy.engine.Engine, drop_if_exists : bool = False): if drop_if_exists: _Base.metadata.drop_all(db_engine) _Base.metadata.create_all(db_engine) - # create_performance_enhancers(db_engine) -- GitLab From b9333b2d333c661be4b9aaba5a294be42667a2a4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 12:35:38 +0000 Subject: [PATCH 16/16] Pre-merge code cleanup --- my_deploy.sh | 2 +- src/qos_profile/.gitlab-ci.yml | 16 +--------------- src/qos_profile/tests/conftest.py | 2 +- src/qos_profile/tests/test_constraints.py | 2 +- src/qos_profile/tests/test_crud.py | 2 +- 5 files changed, 5 insertions(+), 19 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index 6d0a488c2..344ca44ee 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml index f46d63856..0525e69b3 100644 --- a/src/qos_profile/.gitlab-ci.yml +++ b/src/qos_profile/.gitlab-ci.yml @@ -53,12 +53,10 @@ unit_test qos_profile: # QoSProfile-related - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - - docker pull "nats:2.9" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" # environment preparation @@ -68,26 +66,17 @@ unit_test qos_profile: --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 --volume "crdb:/cockroach/cockroach-data" cockroachdb/cockroach:latest-v22.2 start-single-node - - > - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 - nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - docker logs crdb - - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - docker logs nats - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $NATS_ADDRESS - > # QoSProfile preparation - > docker run --name $IMAGE_NAME -d -p 3030:3030 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG @@ -108,10 +97,7 @@ unit_test qos_profile: # Check status after the tests - docker ps -a - docker logs $IMAGE_NAME - - - docker rm -f $IMAGE_NAME - - - docker rm -f $IMAGE_NAME crdb nats + - docker rm -f $IMAGE_NAME crdb - docker volume rm -f crdb - docker network rm teraflowbridge - docker volume prune --force diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 7c8424e00..8983183f3 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -45,4 +45,4 @@ def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: qos_profile.packetDelayBudget.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['packetDelayBudget'])) qos_profile.jitter.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['jitter'])) qos_profile.packetErrorLossRate = qos_profile_data['packetErrorLossRate'] - return qos_profile \ No newline at end of file + return qos_profile diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py index dd03ff7ae..78fe73d64 100644 --- a/src/qos_profile/tests/test_constraints.py +++ b/src/qos_profile/tests/test_constraints.py @@ -91,4 +91,4 @@ def test_get_constraints(qos_profile_client: QoSProfileClient): assert constraint_2.schedule.start_timestamp == 1726063284.25332 assert constraint_2.schedule.duration_days == 1 - qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) \ No newline at end of file + qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index 1037f3849..9b92646c3 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -114,4 +114,4 @@ def test_failed_delete_qos_profiles(qos_profile_client: QoSProfileClient): def test_delete_qos_profiles(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) - assert qos_profiles_deleted == Empty() \ No newline at end of file + assert qos_profiles_deleted == Empty() -- GitLab