From 9b7ed9ee82b7dfc2cb8505839b32d030f24e2adb Mon Sep 17 00:00:00 2001
From: hajipour <shajipour@cttc.es>
Date: Fri, 27 Sep 2024 11:24:22 +0200
Subject: [PATCH] feat: QoSProfile database moved to logical database in CRDB
and context interaction removed from QoSProfile component: - tests updated -
qos_profile constraint aded to ConstraintKindEnum and Constraint parser of
context database - GenericDatabase class used to create qos_profile database
- qos_profile component activation by default added to my_deploy.sh
---
.gitignore | 1 +
deploy/crdb.sh | 3 ++
deploy/tfs.sh | 3 +-
my_deploy.sh | 6 ++-
proto/qos_profile.proto | 19 +++----
src/context/service/database/Constraint.py | 3 +-
.../database/models/ConstraintModel.py | 1 +
src/qos_profile/Dockerfile | 8 ---
src/qos_profile/client/QoSProfileClient.py | 5 +-
.../service/QoSProfileServiceServicerImpl.py | 38 +++++--------
src/qos_profile/service/__main__.py | 21 ++++----
.../service/database/QoSProfile.py | 4 +-
src/qos_profile/tests/conftest.py | 5 +-
src/qos_profile/tests/test_constraints.py | 54 ++-----------------
src/qos_profile/tests/test_crud.py | 5 +-
15 files changed, 54 insertions(+), 122 deletions(-)
diff --git a/.gitignore b/.gitignore
index e1f87cfd3..a0ac78095 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,7 @@ share/python-wheels/
.installed.cfg
*.egg
MANIFEST
+.my_venv/
# requirements.txt # removed to enable tracking versions of packages over time
# PyInstaller
diff --git a/deploy/crdb.sh b/deploy/crdb.sh
index 3e80b6350..474c32ef7 100755
--- a/deploy/crdb.sh
+++ b/deploy/crdb.sh
@@ -171,6 +171,9 @@ function crdb_drop_database_single() {
kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
--execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE};"
+ kubectl exec -i --namespace ${CRDB_NAMESPACE} cockroachdb-0 -- \
+ ./cockroach sql --certs-dir=/cockroach/cockroach-certs --url=${CRDB_CLIENT_URL} \
+ --execute "DROP DATABASE IF EXISTS ${CRDB_DATABASE_QOSPROFILE};"
echo
}
diff --git a/deploy/tfs.sh b/deploy/tfs.sh
index c4f5b106f..e3a1a03d9 100755
--- a/deploy/tfs.sh
+++ b/deploy/tfs.sh
@@ -202,11 +202,10 @@ kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --ty
printf "\n"
echo "Create secret with CockroachDB data for QoSProfile"
-CRDB_DATABASE_QoSProfile="tfs_qos_profile" # TODO: change by specific configurable environment variable
kubectl create secret generic crdb-qos-profile-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
- --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QoSProfile} \
+ --from-literal=CRDB_DATABASE=${CRDB_DATABASE_QOSPROFILE} \
--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
--from-literal=CRDB_SSLMODE=require
diff --git a/my_deploy.sh b/my_deploy.sh
index f0f4e8f6b..60084cc76 100755
--- a/my_deploy.sh
+++ b/my_deploy.sh
@@ -20,8 +20,7 @@
export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/"
# Set the list of components, separated by spaces, you want to build images for, and deploy.
-# export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile"
-export TFS_COMPONENTS="context device pathcomp service nbi qos_profile"
+export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator qos_profile"
# Uncomment to activate Monitoring (old)
#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"
@@ -117,6 +116,9 @@ export CRDB_PASSWORD="tfs123"
# Set the database name to be used by Context.
export CRDB_DATABASE="tfs"
+# Set the database to be used by the QoSProfile component
+export CRDB_DATABASE_QOSPROFILE="tfs_qos_profile"
+
# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing.
# See ./deploy/all.sh or ./deploy/crdb.sh for additional details
export CRDB_DEPLOY_MODE="single"
diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto
index 1237314e1..dc8348766 100644
--- a/proto/qos_profile.proto
+++ b/proto/qos_profile.proto
@@ -17,24 +17,19 @@ package qos_profile;
import "context.proto";
-
-message QoSProfileId {
- context.Uuid qos_profile_id = 1;
-}
-
message QoSProfileValueUnitPair {
int32 value = 1;
string unit = 2;
}
message QoDConstraintsRequest {
- QoSProfileId qos_profile_id = 1;
+ context.QoSProfileId qos_profile_id = 1;
double start_timestamp = 2;
float duration = 3;
}
message QoSProfile {
- QoSProfileId qos_profile_id = 1;
+ context.QoSProfileId qos_profile_id = 1;
string name = 2;
string description = 3;
string status = 4;
@@ -54,10 +49,10 @@ message QoSProfile {
service QoSProfileService {
- rpc CreateQoSProfile (QoSProfile ) returns (QoSProfile ) {}
- rpc UpdateQoSProfile (QoSProfile ) returns (QoSProfile ) {}
- rpc DeleteQoSProfile (QoSProfileId ) returns (context.Empty ) {}
- rpc GetQoSProfile (QoSProfileId ) returns (QoSProfile ) {}
- rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {}
+ rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {}
+ rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {}
+ rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {}
+ rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {}
+ rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {}
rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint ) {}
}
diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py
index db96ed9de..0b0422192 100644
--- a/src/context/service/database/Constraint.py
+++ b/src/context/service/database/Constraint.py
@@ -69,7 +69,8 @@ def compose_constraints_data(
constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid)
elif kind in {
ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY,
- ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS
+ ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS,
+ ConstraintKindEnum.QOS_PROFILE
}:
constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value)
else:
diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py
index fc56a1145..3eef030fc 100644
--- a/src/context/service/database/models/ConstraintModel.py
+++ b/src/context/service/database/models/ConstraintModel.py
@@ -31,6 +31,7 @@ class ConstraintKindEnum(enum.Enum):
SLA_LATENCY = 'sla_latency'
SLA_AVAILABILITY = 'sla_availability'
SLA_ISOLATION = 'sla_isolation'
+ QOS_PROFILE = 'qos_profile'
EXCLUSIONS = 'exclusions'
class ServiceConstraintModel(_Base):
diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile
index d8576ac49..361dc588c 100644
--- a/src/qos_profile/Dockerfile
+++ b/src/qos_profile/Dockerfile
@@ -62,14 +62,6 @@ RUN python3 -m pip install -r requirements.txt
# Add component files into working directory
WORKDIR /var/teraflow
-COPY src/context/__init__.py context/__init__.py
-COPY src/context/client/. context/client/
-COPY src/device/__init__.py device/__init__.py
-COPY src/device/client/. device/client/
-COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py
-COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/
-COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py
-COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/
COPY src/qos_profile/. qos_profile/
# Start the service
diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py
index d243323a1..c6ca46dce 100644
--- a/src/qos_profile/client/QoSProfileClient.py
+++ b/src/qos_profile/client/QoSProfileClient.py
@@ -16,8 +16,9 @@ from typing import Iterator
import grpc, logging
from common.Constants import ServiceNameEnum
from common.Settings import get_service_host, get_service_port_grpc
-from common.proto.context_pb2 import Empty
-from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest
+from common.proto.context_pb2 import Empty, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest
+from common.proto.context_pb2 import Constraint
from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub
from common.tools.client.RetryDecorator import retry, delay_exponential
from common.tools.grpc.Tools import grpc_message_to_json_string
diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py
index 860ee2ee3..47f5fbb25 100644
--- a/src/qos_profile/service/QoSProfileServiceServicerImpl.py
+++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py
@@ -17,10 +17,9 @@ from typing import Iterator
import grpc._channel
from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
-from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty
-from common.proto.qos_profile_pb2 import QoSProfile, QoSProfileId, QoDConstraintsRequest
+from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest
from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer
-from context.client.ContextClient import ContextClient
from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles
@@ -35,7 +34,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile:
- qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
+ qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
if qos_profile is not None:
context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists')
context.set_code(grpc.StatusCode.ALREADY_EXISTS)
@@ -44,7 +43,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile:
- qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
+ qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
if qos_profile is None:
context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
context.set_code(grpc.StatusCode.NOT_FOUND)
@@ -55,7 +54,7 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty:
qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid)
if qos_profile is None:
- context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
+ context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found')
context.set_code(grpc.StatusCode.NOT_FOUND)
return QoSProfile()
return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid)
@@ -76,28 +75,15 @@ class QoSProfileServiceServicerImpl(QoSProfileServiceServicer):
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]:
- context_client = ContextClient()
- try:
- qos_profile = context_client.GetQoSProfile(request.qos_profile_id)
- except grpc._channel._InactiveRpcError as exc:
- if exc.code() == grpc.StatusCode.NOT_FOUND:
- context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
- context.set_code(grpc.StatusCode.NOT_FOUND)
- yield Constraint()
+ qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid)
+ if qos_profile is None:
+ context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found')
+ context.set_code(grpc.StatusCode.NOT_FOUND)
+ yield Constraint()
qos_profile_constraint = Constraint_QoSProfile()
- qos_profile_constraint.target_min_upstream_rate.CopyFrom(qos_profile.targetMinUpstreamRate)
- qos_profile_constraint.max_upstream_rate.CopyFrom(qos_profile.maxUpstreamRate)
- qos_profile_constraint.max_upstream_burst_rate.CopyFrom(qos_profile.maxUpstreamBurstRate)
- qos_profile_constraint.target_min_downstream_rate.CopyFrom(qos_profile.targetMinDownstreamRate)
- qos_profile_constraint.max_downstream_rate.CopyFrom(qos_profile.maxDownstreamRate)
- qos_profile_constraint.max_downstream_burst_rate.CopyFrom(qos_profile.maxDownstreamBurstRate)
- qos_profile_constraint.min_duration.CopyFrom(qos_profile.minDuration)
- qos_profile_constraint.max_duration.CopyFrom(qos_profile.maxDuration)
- qos_profile_constraint.priority = qos_profile.priority
- qos_profile_constraint.packet_delay_budget.CopyFrom(qos_profile.packetDelayBudget)
- qos_profile_constraint.jitter.CopyFrom(qos_profile.jitter)
- qos_profile_constraint.packet_error_loss_rate =qos_profile.packetErrorLossRate
+ qos_profile_constraint.qos_profile_name = qos_profile.name
+ qos_profile_constraint.qos_profile_id.CopyFrom(qos_profile.qos_profile_id)
constraint_qos = Constraint()
constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET
constraint_qos.qos_profile.CopyFrom(qos_profile_constraint)
diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py
index 6666d9f19..d734d5567 100644
--- a/src/qos_profile/service/__main__.py
+++ b/src/qos_profile/service/__main__.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import logging, signal, sys, threading
from prometheus_client import start_http_server
from common.Constants import ServiceNameEnum
@@ -19,7 +20,8 @@ from common.Settings import (
ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port,
wait_for_environment_variables
)
-from .database.Engine import Engine
+from common.tools.database.GenericDatabase import Database
+from qos_profile.service.database.models.QoSProfile import QoSProfileModel
from .QoSProfileService import QoSProfileService
terminate = threading.Event()
@@ -50,20 +52,17 @@ def main():
metrics_port = get_metrics_port()
start_http_server(metrics_port)
- # Get Database Engine instance and initialize database, if needed
- LOGGER.info('Getting SQLAlchemy DB Engine...')
- db_engine = Engine.get_engine()
- if db_engine is None:
- LOGGER.error('Unable to get SQLAlchemy DB Engine...')
- return -1
+ db_manager = Database(db_name=os.getenv('CRDB_DATABASE'), model=QoSProfileModel)
try:
- Engine.create_database(db_engine)
- except: # pylint: disable=bare-except # pragma: no cover
- LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_engine.url)))
+ db_manager.create_database()
+ db_manager.create_tables()
+ except Exception as e: # pylint: disable=bare-except # pragma: no cover
+ LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url)))
+ raise e
# Starting service service
- grpc_service = QoSProfileService(db_engine)
+ grpc_service = QoSProfileService(db_manager.db_engine)
grpc_service.start()
# Wait for Ctrl+C or termination signal
diff --git a/src/qos_profile/service/database/QoSProfile.py b/src/qos_profile/service/database/QoSProfile.py
index 33f6cc6ec..86823c165 100644
--- a/src/qos_profile/service/database/QoSProfile.py
+++ b/src/qos_profile/service/database/QoSProfile.py
@@ -19,8 +19,8 @@ from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy_cockroachdb import run_transaction
from typing import List, Optional
-from common.proto.context_pb2 import Empty, Uuid, QoSProfileId, QoSProfileValueUnitPair, QoSProfile
-from common.method_wrappers.ServiceExceptions import NotFoundException
+from common.proto.context_pb2 import Empty, Uuid, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile
from common.tools.grpc.Tools import grpc_message_to_json
from .models.QoSProfile import QoSProfileModel
diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py
index 1f823253f..7c8424e00 100644
--- a/src/qos_profile/tests/conftest.py
+++ b/src/qos_profile/tests/conftest.py
@@ -14,9 +14,8 @@
import pytest
from qos_profile.client.QoSProfileClient import QoSProfileClient
-from common.proto.context_pb2 import Uuid, QoSProfileValueUnitPair, QoSProfileId, QoSProfile
-from common.proto.context_pb2 import Uuid
-from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfileId, QoSProfile
+from common.proto.context_pb2 import Uuid, QoSProfileId
+from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile
@pytest.fixture(scope='function')
def qos_profile_client():
diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py
index 51f160fc5..dd03ff7ae 100644
--- a/src/qos_profile/tests/test_constraints.py
+++ b/src/qos_profile/tests/test_constraints.py
@@ -15,7 +15,7 @@
import logging
from google.protobuf.json_format import MessageToDict
-from common.proto.context_pb2 import QoDConstraintsRequest
+from common.proto.qos_profile_pb2 import QoDConstraintsRequest
from common.tools.grpc.Tools import grpc_message_to_json_string
from qos_profile.client.QoSProfileClient import QoSProfileClient
@@ -73,54 +73,6 @@ qos_profile_data = {
"packetErrorLossRate": 3
}
-target_qos_profile_constraint = {
- "action": "CONSTRAINTACTION_SET",
- "qos_profile": {
- "target_min_upstream_rate": {
- "value": 5,
- "unit": "bps"
- },
- "max_upstream_rate": {
- "value": 5,
- "unit": "bps"
- },
- "max_upstream_burst_rate": {
- "value": 5,
- "unit": "bps"
- },
- "target_min_downstream_rate": {
- "value": 5,
- "unit": "bps"
- },
- "max_downstream_rate": {
- "value": 5,
- "unit": "bps"
- },
- "max_downstream_burst_rate": {
- "value": 5,
- "unit": "bps"
- },
- "min_duration": {
- "value": 5,
- "unit": "Minutes"
- },
- "max_duration": {
- "value": 6,
- "unit": "Minutes"
- },
- "priority": 5,
- "packet_delay_budget": {
- "value": 5,
- "unit": "Minutes"
- },
- "jitter": {
- "value": 5,
- "unit": "Minutes"
- },
- "packet_error_loss_rate": 3
- }
-}
-
def test_get_constraints(qos_profile_client: QoSProfileClient):
qos_profile = create_qos_profile_from_json(qos_profile_data)
@@ -133,8 +85,8 @@ def test_get_constraints(qos_profile_client: QoSProfileClient):
constraint_2 = constraints[1]
assert len(constraints) == 2
assert constraint_1.WhichOneof('constraint') == 'qos_profile'
- print(MessageToDict(constraint_1, preserving_proto_field_name=True))
- assert MessageToDict(constraint_1, preserving_proto_field_name=True) == target_qos_profile_constraint
+ assert constraint_1.qos_profile.qos_profile_id == qos_profile.qos_profile_id
+ assert constraint_1.qos_profile.qos_profile_name == 'QCI_2_voice'
assert constraint_2.WhichOneof('constraint') == 'schedule'
assert constraint_2.schedule.start_timestamp == 1726063284.25332
assert constraint_2.schedule.duration_days == 1
diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py
index e60f34933..1037f3849 100644
--- a/src/qos_profile/tests/test_crud.py
+++ b/src/qos_profile/tests/test_crud.py
@@ -16,7 +16,6 @@ from grpc import RpcError, StatusCode
import logging, pytest
from .conftest import create_qos_profile_from_json
from common.proto.context_pb2 import Empty, Uuid, QoSProfileId
-from common.proto.qos_profile_pb2 import QoSProfileId
from common.tools.grpc.Tools import grpc_message_to_json_string
from qos_profile.client.QoSProfileClient import QoSProfileClient
@@ -94,8 +93,10 @@ def test_get_qos_profile(qos_profile_client: QoSProfileClient):
def test_get_qos_profiles(qos_profile_client: QoSProfileClient):
qos_profile = create_qos_profile_from_json(qos_profile_data)
qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty()))
+ the_qos_profile = [q for q in qos_profiles_got if q.qos_profile_id == qos_profile.qos_profile_id]
LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got)))
- assert qos_profile == qos_profiles_got[0]
+ assert len(the_qos_profile) == 1
+ assert qos_profile == the_qos_profile[0]
def test_update_qos_profile(qos_profile_client: QoSProfileClient):
qos_profile = create_qos_profile_from_json(qos_profile_data)
--
GitLab