feat: QoSProfile database moved to logical database in CRDB and context...

feat: QoSProfile database moved to logical database in CRDB and context interaction removed from QoSProfile component:
- tests updated
- qos_profile constraint aded to ConstraintKindEnum and Constraint parser of context database
- GenericDatabase class used to create qos_profile database
- qos_profile component activation by default added to my_deploy.sh

.gitignore

@@ -25,6 +25,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+.my_venv/
 # requirements.txt  # removed to enable tracking versions of packages over time
 
 # PyInstaller

deploy/crdb.sh

@@ -171,6 +171,9 @@ function crdb_drop_database_single() {
     kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
         ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
         --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
+    kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
+        ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
+        --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE_QOSPROFILE};"
     echo
 }
 

deploy/tfs.sh

@@ -202,11 +202,10 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty
 printf "\n"
 
 echo "Create secret with CockroachDB data for QoSProfile"
-CRDB_DATABASE_QoSProfile="tfs_qos_profile"  # TODO: change by specific configurable environment variable
 kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
     --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
     --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
-    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QoSProfile} \
+    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QOSPROFILE} \
     --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
     --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
     --from-literal=CRDB_SSLMODE=require

my_deploy.sh

@@ -20,8 +20,7 @@
 export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/"
 
 # Set the list of components, separated by spaces, you want to build images for, and deploy.
-# export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile"
-export TFS_COMPONENTS="context device pathcomp service nbi qos_profile"
+export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile"
 
 # Uncomment to activate Monitoring (old)
 #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"
@@ -117,6 +116,9 @@ export CRDB_PASSWORD="tfs123"
 # Set the database name to be used by Context.
 export CRDB_DATABASE="tfs"
 
+# Set the database to be used by the QoSProfile component
+export CRDB_DATABASE_QOSPROFILE="tfs_qos_profile"
+
 # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing.
 # See ./deploy/all.sh or ./deploy/crdb.sh for additional details
 export CRDB_DEPLOY_MODE="single"

proto/qos_profile.proto

@@ -17,24 +17,19 @@ package qos_profile;
 
 import "context.proto";
 
-
-message QoSProfileId {
-  context.Uuid qos_profile_id = 1;
-}
-
 message QoSProfileValueUnitPair {
   int32 value = 1;
   string unit = 2;
 }
 
 message QoDConstraintsRequest {
-  QoSProfileId qos_profile_id = 1;
+  context.QoSProfileId qos_profile_id = 1;
   double start_timestamp = 2;
   float duration = 3;
 }
 
 message QoSProfile {
-  QoSProfileId qos_profile_id = 1;
+  context.QoSProfileId qos_profile_id = 1;
   string name = 2;
   string description = 3;
   string status = 4;
@@ -54,10 +49,10 @@ message QoSProfile {
 
 
 service QoSProfileService {
-  rpc CreateQoSProfile                (QoSProfile           ) returns (QoSProfile         ) {}
-  rpc UpdateQoSProfile                (QoSProfile           ) returns (QoSProfile         ) {}
-  rpc DeleteQoSProfile                (QoSProfileId         ) returns (context.Empty              ) {}
-  rpc GetQoSProfile                   (QoSProfileId         ) returns (QoSProfile         ) {}
-  rpc GetQoSProfiles                  (context.Empty                ) returns (stream QoSProfile  ) {}
+  rpc CreateQoSProfile                (QoSProfile           ) returns (       QoSProfile                 ) {}
+  rpc UpdateQoSProfile                (QoSProfile           ) returns (       QoSProfile                 ) {}
+  rpc DeleteQoSProfile                (context.QoSProfileId ) returns (       context.Empty              ) {}
+  rpc GetQoSProfile                   (context.QoSProfileId ) returns (       QoSProfile                 ) {}
+  rpc GetQoSProfiles                  (context.Empty        ) returns (stream QoSProfile          ) {}
   rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint  ) {}
 }

src/context/service/database/Constraint.py

@@ -69,7 +69,8 @@ def compose_constraints_data(
             constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid)
         elif kind in {
             ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY,
-            ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS
+            ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS,
+            ConstraintKindEnum.QOS_PROFILE
         }:
             constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value)
         else:

src/context/service/database/models/ConstraintModel.py

@@ -31,6 +31,7 @@ class ConstraintKindEnum(enum.Enum):
     SLA_LATENCY       = 'sla_latency'
     SLA_AVAILABILITY  = 'sla_availability'
     SLA_ISOLATION     = 'sla_isolation'
+    QOS_PROFILE       = 'qos_profile'
     EXCLUSIONS        = 'exclusions'
 
 class ServiceConstraintModel(_Base):

src/qos_profile/Dockerfile

@@ -62,14 +62,6 @@ RUN python3 -m pip install -r requirements.txt
 
 # Add component files into working directory
 WORKDIR /var/teraflow
-COPY src/context/__init__.py context/__init__.py
-COPY src/context/client/. context/client/
-COPY src/device/__init__.py device/__init__.py
-COPY src/device/client/. device/client/
-COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py
-COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/
-COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py
-COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/
 COPY src/qos_profile/. qos_profile/
 
 # Start the service

src/qos_profile/client/QoSProfileClient.py

@@ -16,8 +16,9 @@ from typing import Iterator
 import grpc, logging
 from common.Constants import ServiceNameEnum
 from common.Settings import get_service_host, get_service_port_grpc
-from common.proto.context_pb2 import Empty
-from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest
+from common.proto.context_pb2 import Empty, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest
+from common.proto.context_pb2 import Constraint
 from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub
 from common.tools.client.RetryDecorator import retry, delay_exponential
 from common.tools.grpc.Tools import grpc_message_to_json_string

src/qos_profile/service/QoSProfileServiceServicerImpl.py

@@ -17,10 +17,9 @@ from typing import Iterator
 
 import grpc._channel
 from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
-from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty
-from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest
+from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest
 from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer
-from context.client.ContextClient import ContextClient
 from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles
 
 
@@ -35,7 +34,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
 
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile:
-        qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
+        qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
         if qos_profile is not None:
             context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists')
             context.set_code(grpc.StatusCode.ALREADY_EXISTS)
@@ -44,7 +43,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
 
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile:
-        qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
+        qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
         if qos_profile is None:
             context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
             context.set_code(grpc.StatusCode.NOT_FOUND)
@@ -55,7 +54,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
     def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty:
         qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
         if qos_profile is None:
-            context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
+            context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found')
             context.set_code(grpc.StatusCode.NOT_FOUND)
             return QoSProfile()
         return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid)
@@ -76,28 +75,15 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
 
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]:
-        context_client = ContextClient()
-        try:
-            qos_profile = context_client.GetQoSProfile(request.qos_profile_id)
-        except grpc._channel._InactiveRpcError as exc:
-            if exc.code() == grpc.StatusCode.NOT_FOUND:
-                context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
-                context.set_code(grpc.StatusCode.NOT_FOUND)
-                yield Constraint()
+        qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
+        if qos_profile is None:
+            context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
+            context.set_code(grpc.StatusCode.NOT_FOUND)
+            yield Constraint()
 
         qos_profile_constraint = Constraint_QoSProfile()
-        qos_profile_constraint.target_min_upstream_rate.CopyFrom(qos_profile.targetMinUpstreamRate)
-        qos_profile_constraint.max_upstream_rate.CopyFrom(qos_profile.maxUpstreamRate)
-        qos_profile_constraint.max_upstream_burst_rate.CopyFrom(qos_profile.maxUpstreamBurstRate)
-        qos_profile_constraint.target_min_downstream_rate.CopyFrom(qos_profile.targetMinDownstreamRate)
-        qos_profile_constraint.max_downstream_rate.CopyFrom(qos_profile.maxDownstreamRate)
-        qos_profile_constraint.max_downstream_burst_rate.CopyFrom(qos_profile.maxDownstreamBurstRate)
-        qos_profile_constraint.min_duration.CopyFrom(qos_profile.minDuration)
-        qos_profile_constraint.max_duration.CopyFrom(qos_profile.maxDuration)
-        qos_profile_constraint.priority = qos_profile.priority
-        qos_profile_constraint.packet_delay_budget.CopyFrom(qos_profile.packetDelayBudget)
-        qos_profile_constraint.jitter.CopyFrom(qos_profile.jitter)
-        qos_profile_constraint.packet_error_loss_rate =qos_profile.packetErrorLossRate
+        qos_profile_constraint.qos_profile_name = qos_profile.name
+        qos_profile_constraint.qos_profile_id.CopyFrom(qos_profile.qos_profile_id)
         constraint_qos = Constraint()
         constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET
         constraint_qos.qos_profile.CopyFrom(qos_profile_constraint)

src/qos_profile/service/__main__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import logging, signal, sys, threading
 from prometheus_client import start_http_server
 from common.Constants import ServiceNameEnum
@@ -19,7 +20,8 @@ from common.Settings import (
     ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port,
     wait_for_environment_variables
 )
-from .database.Engine import Engine
+from common.tools.database.GenericDatabase import Database
+from qos_profile.service.database.models.QoSProfile import QoSProfileModel
 from .QoSProfileService import QoSProfileService
 
 terminate = threading.Event()
@@ -50,20 +52,17 @@ def main():
     metrics_port = get_metrics_port()
     start_http_server(metrics_port)
 
-    # Get Database Engine instance and initialize database, if needed
-    LOGGER.info('Getting SQLAlchemy DB Engine...')
-    db_engine = Engine.get_engine()
-    if db_engine is None:
-        LOGGER.error('Unable to get SQLAlchemy DB Engine...')
-        return -1
+    db_manager = Database(db_name=os.getenv('CRDB_DATABASE'), model=QoSProfileModel)
 
     try:
-        Engine.create_database(db_engine)
-    except: # pylint: disable=bare-except # pragma: no cover
-        LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_engine.url)))
+        db_manager.create_database()
+        db_manager.create_tables()
+    except Exception as e: # pylint: disable=bare-except # pragma: no cover
+        LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url)))
+        raise e
 
     # Starting service service
-    grpc_service = QoSProfileService(db_engine)
+    grpc_service = QoSProfileService(db_manager.db_engine)
     grpc_service.start()
 
     # Wait for Ctrl+C or termination signal

src/qos_profile/service/database/QoSProfile.py

@@ -19,8 +19,8 @@ from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import List, Optional
 
-from common.proto.context_pb2 import Empty, Uuid, QoSProfileId, QoSProfileValueUnitPair, QoSProfile
-from common.method_wrappers.ServiceExceptions import NotFoundException
+from common.proto.context_pb2 import Empty, Uuid, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile
 from common.tools.grpc.Tools import grpc_message_to_json
 from .models.QoSProfile import QoSProfileModel
 

src/qos_profile/tests/conftest.py

@@ -14,9 +14,8 @@
 
 import pytest
 from qos_profile.client.QoSProfileClient import QoSProfileClient
-from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile
-from common.proto.context_pb2 import Uuid
-from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfileId, QoSProfile
+from common.proto.context_pb2 import Uuid, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile
 
 @pytest.fixture(scope='function')
 def qos_profile_client():

src/qos_profile/tests/test_constraints.py

@@ -15,7 +15,7 @@
 import logging
 from google.protobuf.json_format import MessageToDict
 
-from common.proto.context_pb2 import QoDConstraintsRequest
+from common.proto.qos_profile_pb2 import QoDConstraintsRequest
 from common.tools.grpc.Tools import grpc_message_to_json_string
 from qos_profile.client.QoSProfileClient import QoSProfileClient
 
@@ -73,54 +73,6 @@ qos_profile_data = {
   "packetErrorLossRate": 3
 }
 
-target_qos_profile_constraint = {
-  "action": "CONSTRAINTACTION_SET",
-  "qos_profile": {
-    "target_min_upstream_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "max_upstream_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "max_upstream_burst_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "target_min_downstream_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "max_downstream_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "max_downstream_burst_rate": {
-      "value": 5,
-      "unit": "bps"
-    },
-    "min_duration": {
-      "value": 5,
-      "unit": "Minutes"
-    },
-    "max_duration": {
-      "value": 6,
-      "unit": "Minutes"
-    },
-    "priority": 5,
-    "packet_delay_budget": {
-      "value": 5,
-      "unit": "Minutes"
-    },
-    "jitter": {
-      "value": 5,
-      "unit": "Minutes"
-    },
-    "packet_error_loss_rate": 3
-  }
-}
-
 
 def test_get_constraints(qos_profile_client: QoSProfileClient):
     qos_profile = create_qos_profile_from_json(qos_profile_data)
@@ -133,8 +85,8 @@ def test_get_constraints(qos_profile_client: QoSProfileClient):
     constraint_2 = constraints[1]
     assert len(constraints) == 2
     assert constraint_1.WhichOneof('constraint') == 'qos_profile'
-    print(MessageToDict(constraint_1, preserving_proto_field_name=True))
-    assert MessageToDict(constraint_1, preserving_proto_field_name=True) == target_qos_profile_constraint
+    assert constraint_1.qos_profile.qos_profile_id == qos_profile.qos_profile_id
+    assert constraint_1.qos_profile.qos_profile_name == 'QCI_2_voice'
     assert constraint_2.WhichOneof('constraint') == 'schedule'
     assert constraint_2.schedule.start_timestamp == 1726063284.25332
     assert constraint_2.schedule.duration_days == 1

src/qos_profile/tests/test_crud.py

@@ -16,7 +16,6 @@ from grpc import RpcError, StatusCode
 import logging, pytest
 from .conftest import create_qos_profile_from_json
 from common.proto.context_pb2 import Empty, Uuid, QoSProfileId
-from common.proto.qos_profile_pb2 import QoSProfileId
 from common.tools.grpc.Tools import grpc_message_to_json_string
 from qos_profile.client.QoSProfileClient import QoSProfileClient
 
@@ -94,8 +93,10 @@ def test_get_qos_profile(qos_profile_client: QoSProfileClient):
 def test_get_qos_profiles(qos_profile_client: QoSProfileClient):
     qos_profile = create_qos_profile_from_json(qos_profile_data)
     qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty()))
+    the_qos_profile = [q for q in qos_profiles_got if q.qos_profile_id == qos_profile.qos_profile_id]
     LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got)))
-    assert qos_profile == qos_profiles_got[0]
+    assert len(the_qos_profile) == 1
+    assert qos_profile == the_qos_profile[0]
 
 def test_update_qos_profile(qos_profile_client: QoSProfileClient):
     qos_profile = create_qos_profile_from_json(qos_profile_data)