diff --git a/.gitignore b/.gitignore index e1f87cfd3842c264bd219237e9afe113d61c35bc..a0ac78095a9f275ae35060a584c5df2151aa7d0e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.my_venv/ # requirements.txt # removed to enable tracking versions of packages over time # PyInstaller diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 3e80b6350e66ec30a725c45acb7cf954ac3009c8..474c32ef7d644dd57f528fbb5d8e13cc5521a53b 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -171,6 +171,9 @@ function crdb_drop_database_single() { kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};" + kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \ + ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \ + --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE_QOSPROFILE};" echo } diff --git a/deploy/tfs.sh b/deploy/tfs.sh index c4f5b106f95b10764eea93bc893dc8e844459ce4..e3a1a03d9db1388fcba20287c14f009eb740cd65 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -202,11 +202,10 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty printf "\n" echo "Create secret with CockroachDB data for QoSProfile" -CRDB_DATABASE_QoSProfile="tfs_qos_profile" # TODO: change by specific configurable environment variable kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ - --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QoSProfile} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QOSPROFILE} \ --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ --from-literal=CRDB_SSLMODE=require diff --git a/my_deploy.sh b/my_deploy.sh index f0f4e8f6b7edc7034d66c2fb4a1ea50a29b2632d..60084cc76661455792fc81032a5727f7c3036046 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,8 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -# export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" -export TFS_COMPONENTS="context device pathcomp service nbi qos_profile" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -117,6 +116,9 @@ export CRDB_PASSWORD="tfs123" # Set the database name to be used by Context. export CRDB_DATABASE="tfs" +# Set the database to be used by the QoSProfile component +export CRDB_DATABASE_QOSPROFILE="tfs_qos_profile" + # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto index 1237314e17f4cb1b530896fe4de171122d20f01f..dc834876675e5677fa97e8fd62bb07fe5dc7213a 100644 --- a/proto/qos_profile.proto +++ b/proto/qos_profile.proto @@ -17,24 +17,19 @@ package qos_profile; import "context.proto"; - -message QoSProfileId { - context.Uuid qos_profile_id = 1; -} - message QoSProfileValueUnitPair { int32 value = 1; string unit = 2; } message QoDConstraintsRequest { - QoSProfileId qos_profile_id = 1; + context.QoSProfileId qos_profile_id = 1; double start_timestamp = 2; float duration = 3; } message QoSProfile { - QoSProfileId qos_profile_id = 1; + context.QoSProfileId qos_profile_id = 1; string name = 2; string description = 3; string status = 4; @@ -54,10 +49,10 @@ message QoSProfile { service QoSProfileService { - rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {} - rpc DeleteQoSProfile (QoSProfileId ) returns (context.Empty ) {} - rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {} - rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} + rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint ) {} } diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index db96ed9dece96cd5b77412c5d031e7337e360668..0b042219273f4a58e0bfc857ea2df6a3422d94cb 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -69,7 +69,8 @@ def compose_constraints_data( constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid) elif kind in { ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY, - ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS + ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS, + ConstraintKindEnum.QOS_PROFILE }: constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value) else: diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py index fc56a1145983776f1604a8cc6a6b36cbd12370b3..3eef030fccccbe4e4806f12188161bf97018c5f5 100644 --- a/src/context/service/database/models/ConstraintModel.py +++ b/src/context/service/database/models/ConstraintModel.py @@ -31,6 +31,7 @@ class ConstraintKindEnum(enum.Enum): SLA_LATENCY = 'sla_latency' SLA_AVAILABILITY = 'sla_availability' SLA_ISOLATION = 'sla_isolation' + QOS_PROFILE = 'qos_profile' EXCLUSIONS = 'exclusions' class ServiceConstraintModel(_Base): diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile index d8576ac49d7f92e7dbb23c97f4a1e1a4597f6651..361dc588c298b384b597edc2709333ba29cf28de 100644 --- a/src/qos_profile/Dockerfile +++ b/src/qos_profile/Dockerfile @@ -62,14 +62,6 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/__init__.py context/__init__.py -COPY src/context/client/. context/client/ -COPY src/device/__init__.py device/__init__.py -COPY src/device/client/. device/client/ -COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py -COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/ -COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py -COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ COPY src/qos_profile/. qos_profile/ # Start the service diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py index d243323a13b017d37c9509844db7ecfd779df889..c6ca46dce8f47ae5ff993ff0d2789f7ba3136631 100644 --- a/src/qos_profile/client/QoSProfileClient.py +++ b/src/qos_profile/client/QoSProfileClient.py @@ -16,8 +16,9 @@ from typing import Iterator import grpc, logging from common.Constants import ServiceNameEnum from common.Settings import get_service_host, get_service_port_grpc -from common.proto.context_pb2 import Empty -from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest +from common.proto.context_pb2 import Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest +from common.proto.context_pb2 import Constraint from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub from common.tools.client.RetryDecorator import retry, delay_exponential from common.tools.grpc.Tools import grpc_message_to_json_string diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py index 860ee2ee3269d31e4682ceb175b0eac4e0af7730..47f5fbb255d1f11eb0d40b85311ca6bd3185341e 100644 --- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -17,10 +17,9 @@ from typing import Iterator import grpc._channel from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty -from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest +from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer -from context.client.ContextClient import ContextClient from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles @@ -35,7 +34,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) if qos_profile is not None: context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists') context.set_code(grpc.StatusCode.ALREADY_EXISTS) @@ -44,7 +43,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: - qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) if qos_profile is None: context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') context.set_code(grpc.StatusCode.NOT_FOUND) @@ -55,7 +54,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty: qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) if qos_profile is None: - context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') context.set_code(grpc.StatusCode.NOT_FOUND) return QoSProfile() return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) @@ -76,28 +75,15 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]: - context_client = ContextClient() - try: - qos_profile = context_client.GetQoSProfile(request.qos_profile_id) - except grpc._channel._InactiveRpcError as exc: - if exc.code() == grpc.StatusCode.NOT_FOUND: - context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') - context.set_code(grpc.StatusCode.NOT_FOUND) - yield Constraint() + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + yield Constraint() qos_profile_constraint = Constraint_QoSProfile() - qos_profile_constraint.target_min_upstream_rate.CopyFrom(qos_profile.targetMinUpstreamRate) - qos_profile_constraint.max_upstream_rate.CopyFrom(qos_profile.maxUpstreamRate) - qos_profile_constraint.max_upstream_burst_rate.CopyFrom(qos_profile.maxUpstreamBurstRate) - qos_profile_constraint.target_min_downstream_rate.CopyFrom(qos_profile.targetMinDownstreamRate) - qos_profile_constraint.max_downstream_rate.CopyFrom(qos_profile.maxDownstreamRate) - qos_profile_constraint.max_downstream_burst_rate.CopyFrom(qos_profile.maxDownstreamBurstRate) - qos_profile_constraint.min_duration.CopyFrom(qos_profile.minDuration) - qos_profile_constraint.max_duration.CopyFrom(qos_profile.maxDuration) - qos_profile_constraint.priority = qos_profile.priority - qos_profile_constraint.packet_delay_budget.CopyFrom(qos_profile.packetDelayBudget) - qos_profile_constraint.jitter.CopyFrom(qos_profile.jitter) - qos_profile_constraint.packet_error_loss_rate =qos_profile.packetErrorLossRate + qos_profile_constraint.qos_profile_name = qos_profile.name + qos_profile_constraint.qos_profile_id.CopyFrom(qos_profile.qos_profile_id) constraint_qos = Constraint() constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET constraint_qos.qos_profile.CopyFrom(qos_profile_constraint) diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py index 6666d9f193ff46d801299ab8eeef3102a1abeb31..d734d5567444a283da87eaa90ffb0c225f41165e 100644 --- a/src/qos_profile/service/__main__.py +++ b/src/qos_profile/service/__main__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import logging, signal, sys, threading from prometheus_client import start_http_server from common.Constants import ServiceNameEnum @@ -19,7 +20,8 @@ from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, wait_for_environment_variables ) -from .database.Engine import Engine +from common.tools.database.GenericDatabase import Database +from qos_profile.service.database.models.QoSProfile import QoSProfileModel from .QoSProfileService import QoSProfileService terminate = threading.Event() @@ -50,20 +52,17 @@ def main(): metrics_port = get_metrics_port() start_http_server(metrics_port) - # Get Database Engine instance and initialize database, if needed - LOGGER.info('Getting SQLAlchemy DB Engine...') - db_engine = Engine.get_engine() - if db_engine is None: - LOGGER.error('Unable to get SQLAlchemy DB Engine...') - return -1 + db_manager = Database(db_name=os.getenv('CRDB_DATABASE'), model=QoSProfileModel) try: - Engine.create_database(db_engine) - except: # pylint: disable=bare-except # pragma: no cover - LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_engine.url))) + db_manager.create_database() + db_manager.create_tables() + except Exception as e: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url))) + raise e # Starting service service - grpc_service = QoSProfileService(db_engine) + grpc_service = QoSProfileService(db_manager.db_engine) grpc_service.start() # Wait for Ctrl+C or termination signal diff --git a/src/qos_profile/service/database/QoSProfile.py b/src/qos_profile/service/database/QoSProfile.py index 33f6cc6ec937584c2b51e5efd881764095827285..86823c16586bb15db4cfd846c97d141095aa6944 100644 --- a/src/qos_profile/service/database/QoSProfile.py +++ b/src/qos_profile/service/database/QoSProfile.py @@ -19,8 +19,8 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_cockroachdb import run_transaction from typing import List, Optional -from common.proto.context_pb2 import Empty, Uuid, QoSProfileId, QoSProfileValueUnitPair, QoSProfile -from common.method_wrappers.ServiceExceptions import NotFoundException +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile from common.tools.grpc.Tools import grpc_message_to_json from .models.QoSProfile import QoSProfileModel diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py index 1f823253fd7d9e3f88f7735248e5782fa2c08acc..7c8424e00bf9dfb9bc7ffdcd1a99ea918b7746a2 100644 --- a/src/qos_profile/tests/conftest.py +++ b/src/qos_profile/tests/conftest.py @@ -14,9 +14,8 @@ import pytest from qos_profile.client.QoSProfileClient import QoSProfileClient -from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile -from common.proto.context_pb2 import Uuid -from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfileId, QoSProfile +from common.proto.context_pb2 import Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile @pytest.fixture(scope='function') def qos_profile_client(): diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py index 51f160fc5a21fa3f914872f356825a08e653a6b5..dd03ff7ae23a69f72e489caaf16f0bbd54eadb55 100644 --- a/src/qos_profile/tests/test_constraints.py +++ b/src/qos_profile/tests/test_constraints.py @@ -15,7 +15,7 @@ import logging from google.protobuf.json_format import MessageToDict -from common.proto.context_pb2 import QoDConstraintsRequest +from common.proto.qos_profile_pb2 import QoDConstraintsRequest from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient @@ -73,54 +73,6 @@ qos_profile_data = { "packetErrorLossRate": 3 } -target_qos_profile_constraint = { - "action": "CONSTRAINTACTION_SET", - "qos_profile": { - "target_min_upstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_upstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_upstream_burst_rate": { - "value": 5, - "unit": "bps" - }, - "target_min_downstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_downstream_rate": { - "value": 5, - "unit": "bps" - }, - "max_downstream_burst_rate": { - "value": 5, - "unit": "bps" - }, - "min_duration": { - "value": 5, - "unit": "Minutes" - }, - "max_duration": { - "value": 6, - "unit": "Minutes" - }, - "priority": 5, - "packet_delay_budget": { - "value": 5, - "unit": "Minutes" - }, - "jitter": { - "value": 5, - "unit": "Minutes" - }, - "packet_error_loss_rate": 3 - } -} - def test_get_constraints(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) @@ -133,8 +85,8 @@ def test_get_constraints(qos_profile_client: QoSProfileClient): constraint_2 = constraints[1] assert len(constraints) == 2 assert constraint_1.WhichOneof('constraint') == 'qos_profile' - print(MessageToDict(constraint_1, preserving_proto_field_name=True)) - assert MessageToDict(constraint_1, preserving_proto_field_name=True) == target_qos_profile_constraint + assert constraint_1.qos_profile.qos_profile_id == qos_profile.qos_profile_id + assert constraint_1.qos_profile.qos_profile_name == 'QCI_2_voice' assert constraint_2.WhichOneof('constraint') == 'schedule' assert constraint_2.schedule.start_timestamp == 1726063284.25332 assert constraint_2.schedule.duration_days == 1 diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py index e60f34933c8eefc508f33234f7e399b2ecd6aaee..1037f3849885f6976ccba658c54189de15838415 100644 --- a/src/qos_profile/tests/test_crud.py +++ b/src/qos_profile/tests/test_crud.py @@ -16,7 +16,6 @@ from grpc import RpcError, StatusCode import logging, pytest from .conftest import create_qos_profile_from_json from common.proto.context_pb2 import Empty, Uuid, QoSProfileId -from common.proto.qos_profile_pb2 import QoSProfileId from common.tools.grpc.Tools import grpc_message_to_json_string from qos_profile.client.QoSProfileClient import QoSProfileClient @@ -94,8 +93,10 @@ def test_get_qos_profile(qos_profile_client: QoSProfileClient): def test_get_qos_profiles(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data) qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty())) + the_qos_profile = [q for q in qos_profiles_got if q.qos_profile_id == qos_profile.qos_profile_id] LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got))) - assert qos_profile == qos_profiles_got[0] + assert len(the_qos_profile) == 1 + assert qos_profile == the_qos_profile[0] def test_update_qos_profile(qos_profile_client: QoSProfileClient): qos_profile = create_qos_profile_from_json(qos_profile_data)