diff --git a/.gitignore b/.gitignore index e1f87cfd3842c264bd219237e9afe113d61c35bc..a0ac78095a9f275ae35060a584c5df2151aa7d0e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.my_venv/ # requirements.txt # removed to enable tracking versions of packages over time # PyInstaller diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edb7d9799a2aa2050636dc61f470bfb599442b7a..dcde3bb15412b6a4e48faf19bf247f4570d216f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,7 @@ include: - local: '/src/kpi_value_writer/.gitlab-ci.yml' - local: '/src/telemetry/.gitlab-ci.yml' - local: '/src/analytics/.gitlab-ci.yml' + - local: '/src/qos_profile/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/manifests/qos_profileservice.yaml b/manifests/qos_profileservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..801607880bbcd9a51bacbec396f797dda7132d81 --- /dev/null +++ b/manifests/qos_profileservice.yaml @@ -0,0 +1,101 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: qos-profileservice +spec: + selector: + matchLabels: + app: qos-profileservice + #replicas: 1 + template: + metadata: + annotations: + config.linkerd.io/skip-outbound-ports: "4222" + labels: + app: qos-profileservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/qos_profile:latest + imagePullPolicy: Always + ports: + - containerPort: 20040 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + - name: CRDB_DATABASE + value: "tfs_qos_profile" + envFrom: + - secretRef: + name: crdb-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:20040"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:20040"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: qos-profileservice + labels: + app: qos-profileservice +spec: + type: ClusterIP + selector: + app: qos-profileservice + ports: + - name: grpc + protocol: TCP + port: 20040 + targetPort: 20040 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: qos-profileservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: qos-profileservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/my_deploy.sh b/my_deploy.sh index d6ed6259ae0299f112d62f62833badc8fd09eb13..344ca44ee335e73dcc7b8f8c9ca71ead7d90880f 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -28,6 +28,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate Monitoring Framework (new) #export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" +# Uncomment to activate QoS Profiles +#export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" + # Uncomment to activate BGP-LS Speaker #export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" diff --git a/proto/context.proto b/proto/context.proto index fa9b1959b54746a6b9a426215874840e1cda8d10..85972d956a93dfe09ec9a955cf304c8b3e298bb3 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -536,7 +536,7 @@ message Constraint_Custom { } message Constraint_Schedule { - float start_timestamp = 1; + double start_timestamp = 1; float duration_days = 2; } @@ -599,6 +599,16 @@ message Constraint_Exclusions { repeated LinkId link_ids = 4; } + +message QoSProfileId { + context.Uuid qos_profile_id = 1; +} + +message Constraint_QoSProfile { + QoSProfileId qos_profile_id = 1; + string qos_profile_name = 2; +} + message Constraint { ConstraintActionEnum action = 1; oneof constraint { @@ -611,6 +621,7 @@ message Constraint { Constraint_SLA_Availability sla_availability = 8; Constraint_SLA_Isolation_level sla_isolation = 9; Constraint_Exclusions exclusions = 10; + Constraint_QoSProfile qos_profile = 11; } } diff --git a/proto/qos_profile.proto b/proto/qos_profile.proto new file mode 100644 index 0000000000000000000000000000000000000000..d032addf4889c8a7a19c260c23df6c74c8ffe55b --- /dev/null +++ b/proto/qos_profile.proto @@ -0,0 +1,58 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package qos_profile; + +import "context.proto"; + +message QoSProfileValueUnitPair { + int32 value = 1; + string unit = 2; +} + +message QoDConstraintsRequest { + context.QoSProfileId qos_profile_id = 1; + double start_timestamp = 2; + float duration = 3; +} + +message QoSProfile { + context.QoSProfileId qos_profile_id = 1; + string name = 2; + string description = 3; + string status = 4; + QoSProfileValueUnitPair targetMinUpstreamRate = 5; + QoSProfileValueUnitPair maxUpstreamRate = 6; + QoSProfileValueUnitPair maxUpstreamBurstRate = 7; + QoSProfileValueUnitPair targetMinDownstreamRate = 8; + QoSProfileValueUnitPair maxDownstreamRate = 9; + QoSProfileValueUnitPair maxDownstreamBurstRate = 10; + QoSProfileValueUnitPair minDuration = 11; + QoSProfileValueUnitPair maxDuration = 12; + int32 priority = 13; + QoSProfileValueUnitPair packetDelayBudget = 14; + QoSProfileValueUnitPair jitter = 15; + int32 packetErrorLossRate = 16; +} + + +service QoSProfileService { + rpc CreateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc UpdateQoSProfile (QoSProfile ) returns ( QoSProfile ) {} + rpc DeleteQoSProfile (context.QoSProfileId ) returns ( context.Empty ) {} + rpc GetQoSProfile (context.QoSProfileId ) returns ( QoSProfile ) {} + rpc GetQoSProfiles (context.Empty ) returns (stream QoSProfile ) {} + rpc GetConstraintListFromQoSProfile (QoDConstraintsRequest) returns (stream context.Constraint) {} +} diff --git a/scripts/show_logs_qos_profile.sh b/scripts/show_logs_qos_profile.sh new file mode 100755 index 0000000000000000000000000000000000000000..744bed9a62af6c376e94d7d13492b796821ecc53 --- /dev/null +++ b/scripts/show_logs_qos_profile.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qos_profileservice -c server diff --git a/src/common/Constants.py b/src/common/Constants.py index 4fff09ba0d5061a41fa01d6ad44228283d2fca51..33a5d5047c8969118417a8b06a13cb8fb6fbf7df 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -70,6 +70,7 @@ class ServiceNameEnum(Enum): TELEMETRYBACKEND = 'telemetry-backend' ANALYTICSFRONTEND = 'analytics-frontend' ANALYTICSBACKEND = 'analytics-backend' + QOSPROFILE = 'qos-profile' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -100,6 +101,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.OPTICALCONTROLLER .value : 10060, ServiceNameEnum.QKD_APP .value : 10070, ServiceNameEnum.BGPLS .value : 20030, + ServiceNameEnum.QOSPROFILE .value : 20040, ServiceNameEnum.KPIMANAGER .value : 30010, ServiceNameEnum.KPIVALUEAPI .value : 30020, ServiceNameEnum.KPIVALUEWRITER .value : 30030, diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index db96ed9dece96cd5b77412c5d031e7337e360668..0b042219273f4a58e0bfc857ea2df6a3422d94cb 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -69,7 +69,8 @@ def compose_constraints_data( constraint_name = '{:s}:{:s}:{:s}'.format(parent_kind, kind.value, endpoint_uuid) elif kind in { ConstraintKindEnum.SCHEDULE, ConstraintKindEnum.SLA_CAPACITY, ConstraintKindEnum.SLA_LATENCY, - ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS + ConstraintKindEnum.SLA_AVAILABILITY, ConstraintKindEnum.SLA_ISOLATION, ConstraintKindEnum.EXCLUSIONS, + ConstraintKindEnum.QOS_PROFILE }: constraint_name = '{:s}:{:s}:'.format(parent_kind, kind.value) else: diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py index fc56a1145983776f1604a8cc6a6b36cbd12370b3..3eef030fccccbe4e4806f12188161bf97018c5f5 100644 --- a/src/context/service/database/models/ConstraintModel.py +++ b/src/context/service/database/models/ConstraintModel.py @@ -31,6 +31,7 @@ class ConstraintKindEnum(enum.Enum): SLA_LATENCY = 'sla_latency' SLA_AVAILABILITY = 'sla_availability' SLA_ISOLATION = 'sla_isolation' + QOS_PROFILE = 'qos_profile' EXCLUSIONS = 'exclusions' class ServiceConstraintModel(_Base): diff --git a/src/qos_profile/.gitlab-ci.yml b/src/qos_profile/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..0525e69b348eef8cd070ae9fc898a556e1ade680 --- /dev/null +++ b/src/qos_profile/.gitlab-ci.yml @@ -0,0 +1,121 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build qos_profile: + variables: + IMAGE_NAME: "qos_profile" # name of the microservice + IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test qos_profile: + variables: + IMAGE_NAME: "qos_profile" # name of the microservice + IMAGE_TAG: "latest" # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build qos_profile + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + + # QoSProfile-related + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + + script: + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + + # environment preparation + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - > + # QoSProfile preparation + - > + docker run --name $IMAGE_NAME -d -p 3030:3030 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + + # Check status before the tests + - sleep 5 + - docker ps -a + - docker logs $IMAGE_NAME + + # Run the tests + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_crud.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + # Check status after the tests + - docker ps -a + - docker logs $IMAGE_NAME + - docker rm -f $IMAGE_NAME crdb + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml diff --git a/src/qos_profile/Config.py b/src/qos_profile/Config.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/Config.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/Dockerfile b/src/qos_profile/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..361dc588c298b384b597edc2709333ba29cf28de --- /dev/null +++ b/src/qos_profile/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/qos_profile +WORKDIR /var/teraflow/qos_profile +COPY src/qos_profile/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/qos_profile/. qos_profile/ + +# Start the service +ENTRYPOINT ["python", "-m", "qos_profile.service"] diff --git a/src/qos_profile/__init__.py b/src/qos_profile/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/client/QoSProfileClient.py b/src/qos_profile/client/QoSProfileClient.py new file mode 100644 index 0000000000000000000000000000000000000000..748b3f208cc44e80c2e7b88f163f937328249633 --- /dev/null +++ b/src/qos_profile/client/QoSProfileClient.py @@ -0,0 +1,91 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterator +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.proto.context_pb2 import Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest +from common.proto.context_pb2 import Constraint +from common.proto.qos_profile_pb2_grpc import QoSProfileServiceStub +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class QoSProfileClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.QOSPROFILE) + if not port: port = get_service_port_grpc(ServiceNameEnum.QOSPROFILE) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = QoSProfileServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def CreateQoSProfile(self, request: QoSProfile) -> QoSProfile: + LOGGER.debug('CreateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.CreateQoSProfile(request) + LOGGER.debug('CreateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def UpdateQoSProfile(self, request: QoSProfile) -> QoSProfile: + LOGGER.debug('UpdateQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.UpdateQoSProfile(request) + LOGGER.debug('UpdateQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteQoSProfile(self, request: QoSProfileId) -> Empty: + LOGGER.debug('DeleteQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteQoSProfile(request) + LOGGER.debug('DeleteQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfile(self, request: QoSProfileId) -> QoSProfile: + LOGGER.debug('GetQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfile(request) + LOGGER.debug('GetQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetQoSProfiles(self, request: Empty) -> Iterator[QoSProfile]: + LOGGER.debug('GetQoSProfiles request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetQoSProfiles(request) + LOGGER.debug('GetQoSProfiles result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest) -> Iterator[Constraint]: + LOGGER.debug('GetConstraintListFromQoSProfile request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetConstraintListFromQoSProfile(request) + LOGGER.debug('GetConstraintListFromQoSProfile result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/qos_profile/client/__init__.py b/src/qos_profile/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/requirements.in b/src/qos_profile/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..3e98fef362277dbf60019902e115d1c733bea9e7 --- /dev/null +++ b/src/qos_profile/requirements.in @@ -0,0 +1,18 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* diff --git a/src/qos_profile/service/QoSProfileService.py b/src/qos_profile/service/QoSProfileService.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5c5591b498787240c5390bbe5575822bc9da91 --- /dev/null +++ b/src/qos_profile/service/QoSProfileService.py @@ -0,0 +1,29 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlalchemy +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.qos_profile_pb2_grpc import add_QoSProfileServiceServicer_to_server +from common.tools.service.GenericGrpcService import GenericGrpcService +from .QoSProfileServiceServicerImpl import QoSProfileServiceServicerImpl + +class QoSProfileService(GenericGrpcService): + def __init__(self, db_engine: sqlalchemy.engine.Engine, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.QOSPROFILE) + super().__init__(port, cls_name=cls_name) + self.qos_profile_servicer = QoSProfileServiceServicerImpl(db_engine) + + def install_servicers(self): + add_QoSProfileServiceServicer_to_server(self.qos_profile_servicer, self.server) diff --git a/src/qos_profile/service/QoSProfileServiceServicerImpl.py b/src/qos_profile/service/QoSProfileServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..47f5fbb255d1f11eb0d40b85311ca6bd3185341e --- /dev/null +++ b/src/qos_profile/service/QoSProfileServiceServicerImpl.py @@ -0,0 +1,94 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging, sqlalchemy +from typing import Iterator + +import grpc._channel +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Constraint, ConstraintActionEnum, Constraint_QoSProfile, Constraint_Schedule, Empty, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfile, QoDConstraintsRequest +from common.proto.qos_profile_pb2_grpc import QoSProfileServiceServicer +from .database.QoSProfile import set_qos_profile, delete_qos_profile, get_qos_profile, get_qos_profiles + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool('QoSProfile', 'RPC') + +class QoSProfileServiceServicerImpl(QoSProfileServiceServicer): + def __init__(self, db_engine: sqlalchemy.engine.Engine) -> None: + LOGGER.debug('Servicer Created') + self.db_engine = db_engine + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def CreateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) + if qos_profile is not None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} already exists') + context.set_code(grpc.StatusCode.ALREADY_EXISTS) + return QoSProfile() + return set_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def UpdateQoSProfile(self, request: QoSProfile, context: grpc.ServicerContext) -> QoSProfile: + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return set_qos_profile(self.db_engine, request) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> Empty: + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return delete_qos_profile(self.db_engine, request.qos_profile_id.uuid) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfile(self, request: QoSProfileId, context: grpc.ServicerContext) -> QoSProfile: + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + return QoSProfile() + return qos_profile + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetQoSProfiles(self, request: Empty, context: grpc.ServicerContext) -> Iterator[QoSProfile]: + yield from get_qos_profiles(self.db_engine, request) + + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetConstraintListFromQoSProfile(self, request: QoDConstraintsRequest, context: grpc.ServicerContext) -> Iterator[Constraint]: + qos_profile = get_qos_profile(self.db_engine, request.qos_profile_id.qos_profile_id.uuid) + if qos_profile is None: + context.set_details(f'QoSProfile {request.qos_profile_id.qos_profile_id.uuid} not found') + context.set_code(grpc.StatusCode.NOT_FOUND) + yield Constraint() + + qos_profile_constraint = Constraint_QoSProfile() + qos_profile_constraint.qos_profile_name = qos_profile.name + qos_profile_constraint.qos_profile_id.CopyFrom(qos_profile.qos_profile_id) + constraint_qos = Constraint() + constraint_qos.action = ConstraintActionEnum.CONSTRAINTACTION_SET + constraint_qos.qos_profile.CopyFrom(qos_profile_constraint) + yield constraint_qos + constraint_schedule = Constraint() + constraint_schedule.action = ConstraintActionEnum.CONSTRAINTACTION_SET + constraint_schedule.schedule.CopyFrom(Constraint_Schedule(start_timestamp=request.start_timestamp, duration_days=request.duration/86400)) + yield constraint_schedule diff --git a/src/qos_profile/service/__init__.py b/src/qos_profile/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/service/__main__.py b/src/qos_profile/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f9e6de92b3ddf24e46a53f478bf90046e32d523 --- /dev/null +++ b/src/qos_profile/service/__main__.py @@ -0,0 +1,66 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from prometheus_client import start_http_server +from common.Settings import get_log_level, get_metrics_port +from common.tools.database.GenericDatabase import Database +from .QoSProfileService import QoSProfileService +from .database.models.QoSProfile import QoSProfileModel + +LOG_LEVEL = get_log_level() +logging.basicConfig(level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") +LOGGER = logging.getLogger(__name__) + + +terminate = threading.Event() + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + LOGGER.info('Starting...') + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Get Database Engine instance and initialize database, if needed + db_manager = Database(QoSProfileModel) + + try: + db_manager.create_database() + db_manager.create_tables() + except Exception as e: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to check/create the database: {:s}'.format(str(db_manager.db_engine.url))) + raise e + + # Starting service + grpc_service = QoSProfileService(db_manager.db_engine) + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/qos_profile/service/database/Engine.py b/src/qos_profile/service/database/Engine.py new file mode 100644 index 0000000000000000000000000000000000000000..6ba1a82d0b5790deded242ecde682020a0c785f8 --- /dev/null +++ b/src/qos_profile/service/database/Engine.py @@ -0,0 +1,55 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sqlalchemy, sqlalchemy_utils +from common.Settings import get_setting + +LOGGER = logging.getLogger(__name__) + +APP_NAME = 'tfs' +ECHO = False # true: dump SQL commands and transactions executed +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@qos-profileservice.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' + +class Engine: + @staticmethod + def get_engine() -> sqlalchemy.engine.Engine: + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = get_setting('CRDB_DATABASE') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + + try: + engine = sqlalchemy.create_engine( + crdb_uri, connect_args={'application_name': APP_NAME}, echo=ECHO, future=True) + except: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) + return None + + return engine + + @staticmethod + def create_database(engine : sqlalchemy.engine.Engine) -> None: + if not sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.create_database(engine.url) + + @staticmethod + def drop_database(engine : sqlalchemy.engine.Engine) -> None: + if sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.drop_database(engine.url) diff --git a/src/qos_profile/service/database/QoSProfile.py b/src/qos_profile/service/database/QoSProfile.py new file mode 100644 index 0000000000000000000000000000000000000000..86823c16586bb15db4cfd846c97d141095aa6944 --- /dev/null +++ b/src/qos_profile/service/database/QoSProfile.py @@ -0,0 +1,116 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy_cockroachdb import run_transaction +from typing import List, Optional + +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile +from common.tools.grpc.Tools import grpc_message_to_json +from .models.QoSProfile import QoSProfileModel + +LOGGER = logging.getLogger(__name__) + +def grpc_message_to_qos_table_data(message: QoSProfile) -> dict: + return { + 'qos_profile_id' : message.qos_profile_id.qos_profile_id.uuid, + 'name' : message.name, + 'description' : message.description, + 'status' : message.status, + 'targetMinUpstreamRate' : grpc_message_to_json(message.targetMinUpstreamRate), + 'maxUpstreamRate' : grpc_message_to_json(message.maxUpstreamRate), + 'maxUpstreamBurstRate' : grpc_message_to_json(message.maxUpstreamBurstRate), + 'targetMinDownstreamRate' : grpc_message_to_json(message.targetMinDownstreamRate), + 'maxDownstreamRate' : grpc_message_to_json(message.maxDownstreamRate), + 'maxDownstreamBurstRate' : grpc_message_to_json(message.maxDownstreamBurstRate), + 'minDuration' : grpc_message_to_json(message.minDuration), + 'maxDuration' : grpc_message_to_json(message.maxDuration), + 'priority' : message.priority, + 'packetDelayBudget' : grpc_message_to_json(message.packetDelayBudget), + 'jitter' : grpc_message_to_json(message.jitter), + 'packetErrorLossRate' : message.packetErrorLossRate, + } + +def qos_table_data_to_grpc_message(data: QoSProfileModel) -> QoSProfile: + return QoSProfile( + qos_profile_id = QoSProfileId(qos_profile_id=Uuid(uuid=data.qos_profile_id)), + name = data.name, + description = data.description, + status = data.status, + targetMinUpstreamRate = QoSProfileValueUnitPair(**data.targetMinUpstreamRate), + maxUpstreamRate = QoSProfileValueUnitPair(**data.maxUpstreamRate), + maxUpstreamBurstRate = QoSProfileValueUnitPair(**data.maxUpstreamBurstRate), + targetMinDownstreamRate = QoSProfileValueUnitPair(**data.targetMinDownstreamRate), + maxDownstreamRate = QoSProfileValueUnitPair(**data.maxDownstreamRate), + maxDownstreamBurstRate = QoSProfileValueUnitPair(**data.maxDownstreamBurstRate), + minDuration = QoSProfileValueUnitPair(**data.minDuration), + maxDuration = QoSProfileValueUnitPair(**data.maxDuration), + priority = data.priority, + packetDelayBudget = QoSProfileValueUnitPair(**data.packetDelayBudget), + jitter = QoSProfileValueUnitPair(**data.jitter), + packetErrorLossRate = data.packetErrorLossRate + ) + +def set_qos_profile(db_engine : Engine, request : QoSProfile) -> QoSProfile: + qos_profile_data = grpc_message_to_qos_table_data(request) + def callback(session : Session) -> bool: + stmt = insert(QoSProfileModel).values([qos_profile_data]) + stmt = stmt.on_conflict_do_update(index_elements=[QoSProfileModel.qos_profile_id], + set_=dict( + + name = stmt.excluded.name, + description = stmt.excluded.description, + status = stmt.excluded.status, + targetMinUpstreamRate = stmt.excluded.targetMinUpstreamRate, + maxUpstreamRate = stmt.excluded.maxUpstreamRate, + maxUpstreamBurstRate = stmt.excluded.maxUpstreamBurstRate, + targetMinDownstreamRate = stmt.excluded.targetMinDownstreamRate, + maxDownstreamRate = stmt.excluded.maxDownstreamRate, + maxDownstreamBurstRate = stmt.excluded.maxDownstreamBurstRate, + minDuration = stmt.excluded.minDuration, + maxDuration = stmt.excluded.maxDuration, + priority = stmt.excluded.priority, + packetDelayBudget = stmt.excluded.packetDelayBudget, + jitter = stmt.excluded.jitter, + packetErrorLossRate = stmt.excluded.packetErrorLossRate, + ) + ) + stmt = stmt.returning(QoSProfileModel) + qos_profile = session.execute(stmt).fetchall() + return qos_profile[0] + qos_profile_row = run_transaction(sessionmaker(bind=db_engine), callback) + return qos_table_data_to_grpc_message(qos_profile_row) + +def delete_qos_profile(db_engine : Engine, request : str) -> Empty: + def callback(session : Session) -> bool: + num_deleted = session.query(QoSProfileModel).filter_by(qos_profile_id=request).delete() + return num_deleted > 0 + deleted = run_transaction(sessionmaker(bind=db_engine), callback) + return Empty() + +def get_qos_profile(db_engine : Engine, request : str) -> Optional[QoSProfile]: + def callback(session : Session) -> Optional[QoSProfile]: + obj : Optional[QoSProfileModel] = session.query(QoSProfileModel).filter_by(qos_profile_id=request).one_or_none() + return None if obj is None else qos_table_data_to_grpc_message(obj) + return run_transaction(sessionmaker(bind=db_engine), callback) + +def get_qos_profiles(db_engine : Engine, request : Empty) -> List[QoSProfile]: + def callback(session : Session) -> List[QoSProfile]: + obj_list : List[QoSProfileModel] = session.query(QoSProfileModel).all() + return [qos_table_data_to_grpc_message(obj) for obj in obj_list] + return run_transaction(sessionmaker(bind=db_engine), callback) diff --git a/src/qos_profile/service/database/__init__.py b/src/qos_profile/service/database/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/service/database/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/service/database/models/QoSProfile.py b/src/qos_profile/service/database/models/QoSProfile.py new file mode 100644 index 0000000000000000000000000000000000000000..bfbdeef0a35490b1a62b80bddb098fd2bf90c2e4 --- /dev/null +++ b/src/qos_profile/service/database/models/QoSProfile.py @@ -0,0 +1,38 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy import Column, Integer, String, JSON +from sqlalchemy.dialects.postgresql import UUID +from ._Base import _Base + + +class QoSProfileModel(_Base): + __tablename__ = 'qos_profile' + + qos_profile_id = Column(UUID(as_uuid=False), primary_key=True) + name = Column(String, nullable=False) + description = Column(String, nullable=False) + status = Column(String, nullable=False) + targetMinUpstreamRate = Column(JSON, nullable=False) + maxUpstreamRate = Column(JSON, nullable=False) + maxUpstreamBurstRate = Column(JSON, nullable=False) + targetMinDownstreamRate = Column(JSON, nullable=False) + maxDownstreamRate = Column(JSON, nullable=False) + maxDownstreamBurstRate = Column(JSON, nullable=False) + minDuration = Column(JSON, nullable=False) + maxDuration = Column(JSON, nullable=False) + priority = Column(Integer, nullable=False) + packetDelayBudget = Column(JSON, nullable=False) + jitter = Column(JSON, nullable=False) + packetErrorLossRate = Column(Integer, nullable=False) diff --git a/src/qos_profile/service/database/models/_Base.py b/src/qos_profile/service/database/models/_Base.py new file mode 100644 index 0000000000000000000000000000000000000000..d94dad3cdfc4dad473cc12eb00d502b05595b8f4 --- /dev/null +++ b/src/qos_profile/service/database/models/_Base.py @@ -0,0 +1,22 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlalchemy +from sqlalchemy.orm import declarative_base + +_Base = declarative_base() + +def rebuild_database(db_engine : sqlalchemy.engine.Engine, drop_if_exists : bool = False): + if drop_if_exists: _Base.metadata.drop_all(db_engine) + _Base.metadata.create_all(db_engine) diff --git a/src/qos_profile/service/database/models/__init__.py b/src/qos_profile/service/database/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bbfc943b68af13a11e562abbc8680ade71db8f02 --- /dev/null +++ b/src/qos_profile/service/database/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/qos_profile/tests/.gitignore b/src/qos_profile/tests/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6b97d6fe3ad32f39097745229ab7f547f26ecb12 --- /dev/null +++ b/src/qos_profile/tests/.gitignore @@ -0,0 +1 @@ +# Add here your files containing confidential testbed details such as IP addresses, ports, usernames, passwords, etc. diff --git a/src/qos_profile/tests/__init__.py b/src/qos_profile/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/qos_profile/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qos_profile/tests/conftest.py b/src/qos_profile/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..8983183f3b3fc5da2ffa5a2efb2d2d0bac203f43 --- /dev/null +++ b/src/qos_profile/tests/conftest.py @@ -0,0 +1,48 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from qos_profile.client.QoSProfileClient import QoSProfileClient +from common.proto.context_pb2 import Uuid, QoSProfileId +from common.proto.qos_profile_pb2 import QoSProfileValueUnitPair, QoSProfile + +@pytest.fixture(scope='function') +def qos_profile_client(): + _client = QoSProfileClient(host='0.0.0.0', port=20040) + yield _client + _client.close() + + + +def create_qos_profile_from_json(qos_profile_data: dict) -> QoSProfile: + def create_QoSProfileValueUnitPair(data) -> QoSProfileValueUnitPair: + return QoSProfileValueUnitPair(value=data['value'], unit=data['unit']) + qos_profile = QoSProfile() + qos_profile.qos_profile_id.CopyFrom(QoSProfileId(qos_profile_id=Uuid(uuid=qos_profile_data['qos_profile_id']))) + qos_profile.name = qos_profile_data['name'] + qos_profile.description = qos_profile_data['description'] + qos_profile.status = qos_profile_data['status'] + qos_profile.targetMinUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinUpstreamRate'])) + qos_profile.maxUpstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamRate'])) + qos_profile.maxUpstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxUpstreamBurstRate'])) + qos_profile.targetMinDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['targetMinDownstreamRate'])) + qos_profile.maxDownstreamRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamRate'])) + qos_profile.maxDownstreamBurstRate.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDownstreamBurstRate'])) + qos_profile.minDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['minDuration'])) + qos_profile.maxDuration.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['maxDuration'])) + qos_profile.priority = qos_profile_data['priority'] + qos_profile.packetDelayBudget.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['packetDelayBudget'])) + qos_profile.jitter.CopyFrom(create_QoSProfileValueUnitPair(qos_profile_data['jitter'])) + qos_profile.packetErrorLossRate = qos_profile_data['packetErrorLossRate'] + return qos_profile diff --git a/src/qos_profile/tests/test_constraints.py b/src/qos_profile/tests/test_constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..78fe73d64c11502c6468134f937003d2700e5b71 --- /dev/null +++ b/src/qos_profile/tests/test_constraints.py @@ -0,0 +1,94 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from google.protobuf.json_format import MessageToDict + +from common.proto.qos_profile_pb2 import QoDConstraintsRequest +from common.tools.grpc.Tools import grpc_message_to_json_string +from qos_profile.client.QoSProfileClient import QoSProfileClient + +from .conftest import create_qos_profile_from_json + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +qos_profile_data = { + "qos_profile_id": "0afc905f-f1f0-4ae2-9925-9df17140b8bf", + "name": "QCI_2_voice", + "description": "QoS profile for game streaming", + "status": "ACTIVE", + "targetMinUpstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxUpstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxUpstreamBurstRate": { + "value": 5, + "unit": "bps" + }, + "targetMinDownstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxDownstreamRate": { + "value": 5, + "unit": "bps" + }, + "maxDownstreamBurstRate": { + "value": 5, + "unit": "bps" + }, + "minDuration": { + "value": 5, + "unit": "Minutes" + }, + "maxDuration": { + "value": 6, + "unit": "Minutes" + }, + "priority": 5, + "packetDelayBudget": { + "value": 5, + "unit": "Minutes" + }, + "jitter": { + "value": 5, + "unit": "Minutes" + }, + "packetErrorLossRate": 3 +} + + +def test_get_constraints(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_created))) + constraints = list(qos_profile_client.GetConstraintListFromQoSProfile(QoDConstraintsRequest( + qos_profile_id=qos_profile.qos_profile_id, start_timestamp=1726063284.25332, duration=86400) + )) + constraint_1 = constraints[0] + constraint_2 = constraints[1] + assert len(constraints) == 2 + assert constraint_1.WhichOneof('constraint') == 'qos_profile' + assert constraint_1.qos_profile.qos_profile_id == qos_profile.qos_profile_id + assert constraint_1.qos_profile.qos_profile_name == 'QCI_2_voice' + assert constraint_2.WhichOneof('constraint') == 'schedule' + assert constraint_2.schedule.start_timestamp == 1726063284.25332 + assert constraint_2.schedule.duration_days == 1 + + qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) diff --git a/src/qos_profile/tests/test_crud.py b/src/qos_profile/tests/test_crud.py new file mode 100644 index 0000000000000000000000000000000000000000..9b92646c3341d2801e3b04741430075b4956a263 --- /dev/null +++ b/src/qos_profile/tests/test_crud.py @@ -0,0 +1,117 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from grpc import RpcError, StatusCode +import logging, pytest +from .conftest import create_qos_profile_from_json +from common.proto.context_pb2 import Empty, Uuid, QoSProfileId +from common.tools.grpc.Tools import grpc_message_to_json_string +from qos_profile.client.QoSProfileClient import QoSProfileClient + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +qos_profile_data = { + "qos_profile_id": "f00406f5-8e36-4abc-a0ec-b871c7f062b7", + "name": "QCI_1_voice", + "description": "QoS profile for video streaming", + "status": "ACTIVE", + "targetMinUpstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxUpstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxUpstreamBurstRate": { + "value": 10, + "unit": "bps" + }, + "targetMinDownstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxDownstreamRate": { + "value": 10, + "unit": "bps" + }, + "maxDownstreamBurstRate": { + "value": 10, + "unit": "bps" + }, + "minDuration": { + "value": 12, + "unit": "Minutes" + }, + "maxDuration": { + "value": 12, + "unit": "Minutes" + }, + "priority": 20, + "packetDelayBudget": { + "value": 12, + "unit": "Minutes" + }, + "jitter": { + "value": 12, + "unit": "Minutes" + }, + "packetErrorLossRate": 3 +} + +def test_create_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_created))) + assert qos_profile == qos_profile_created + + +def test_failed_create_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + with pytest.raises(RpcError) as exc: + qos_profile_created = qos_profile_client.CreateQoSProfile(qos_profile) + assert exc.value.code() == StatusCode.ALREADY_EXISTS + +def test_get_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile_got = qos_profile_client.GetQoSProfile(qos_profile.qos_profile_id) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_got))) + assert qos_profile == qos_profile_got + +def test_get_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profiles_got = list(qos_profile_client.GetQoSProfiles(Empty())) + the_qos_profile = [q for q in qos_profiles_got if q.qos_profile_id == qos_profile.qos_profile_id] + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profiles_got))) + assert len(the_qos_profile) == 1 + assert qos_profile == the_qos_profile[0] + +def test_update_qos_profile(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profile.packetErrorLossRate = 5 + qos_profile_updated = qos_profile_client.UpdateQoSProfile(qos_profile) + LOGGER.info('qos_profile_data = {:s}'.format(grpc_message_to_json_string(qos_profile_updated))) + assert qos_profile_updated.packetErrorLossRate == 5 + +def test_failed_delete_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + with pytest.raises(RpcError) as exc: + qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(QoSProfileId(qos_profile_id=Uuid(uuid='f8b1c625-ac01-405c-b1f7-b5ee06e16282'))) + assert exc.value.code() == StatusCode.NOT_FOUND + +def test_delete_qos_profiles(qos_profile_client: QoSProfileClient): + qos_profile = create_qos_profile_from_json(qos_profile_data) + qos_profiles_deleted = qos_profile_client.DeleteQoSProfile(qos_profile.qos_profile_id) + assert qos_profiles_deleted == Empty()