diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e26a1644ffb9ce094c262ecf2f8df6be81985c5..316a38f234c6b54de043bd25eb0b27d8bec86708 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,3 +43,4 @@ include: #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' + - local: '/src/load_generator/.gitlab-ci.yml' diff --git a/manifests/load_generatorservice.yaml b/manifests/load_generatorservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88b1fa397a2708408b7a7ea5b9be28a984e938aa --- /dev/null +++ b/manifests/load_generatorservice.yaml @@ -0,0 +1,67 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: load-generatorservice +spec: + selector: + matchLabels: + app: load-generatorservice + replicas: 1 + template: + metadata: + labels: + app: load-generatorservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: registry.gitlab.com/teraflow-h2020/controller/load_generator:latest + imagePullPolicy: Always + ports: + - containerPort: 50052 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50052"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50052"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: load-generatorservice + labels: + app: load-generatorservice +spec: + type: ClusterIP + selector: + app: load-generatorservice + ports: + - name: grpc + protocol: TCP + port: 50052 + targetPort: 50052 diff --git a/proto/load_generator.proto b/proto/load_generator.proto new file mode 100644 index 0000000000000000000000000000000000000000..00ddb254cd4bdb9e947906f477a408ece079269e --- /dev/null +++ b/proto/load_generator.proto @@ -0,0 +1,23 @@ +// Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package load_generator; + +import "context.proto"; + +service LoadGeneratorService { + rpc Start(context.Empty) returns (context.Empty) {} + rpc Stop (context.Empty) returns (context.Empty) {} +} diff --git a/src/common/Constants.py b/src/common/Constants.py index bdbde21b27f2d18c5ec9cc50fc0d4edec9b22398..bd403c084d388f34fa3dbd0801c69980a9b7ac2f 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -54,7 +54,8 @@ class ServiceNameEnum(Enum): WEBUI = 'webui' # Used for test and debugging only - DLT_GATEWAY = 'dltgateway' + DLT_GATEWAY = 'dltgateway' + LOAD_GENERATOR = 'load_generator' # Default gRPC service ports DEFAULT_SERVICE_GRPC_PORTS = { @@ -72,7 +73,8 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.PATHCOMP .value : 10020, # Used for test and debugging only - ServiceNameEnum.DLT_GATEWAY .value : 50051, + ServiceNameEnum.DLT_GATEWAY .value : 50051, + ServiceNameEnum.LOAD_GENERATOR.value : 50052, } # Default HTTP/REST-API service ports diff --git a/src/common/method_wrappers/tests/deploy_specs.sh b/src/common/method_wrappers/tests/deploy_specs.sh index 238918480ae857e64efb52f652b20ab08a21c2df..ab90ab3d3e7f7b32be59041e5f403fba53f98792 100644 --- a/src/common/method_wrappers/tests/deploy_specs.sh +++ b/src/common/method_wrappers/tests/deploy_specs.sh @@ -7,7 +7,7 @@ export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/" # interdomain slice pathcomp dlt # dbscanserving opticalattackmitigator opticalattackdetector # l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector -export TFS_COMPONENTS="context device pathcomp service slice webui" # automation monitoring compute +export TFS_COMPONENTS="context device pathcomp service slice webui load_generator" # automation monitoring compute # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/load_generator/.gitlab-ci.yml b/src/load_generator/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..a63bd8d0ddcdcff086e6718e2a6ea7feeb337ea1 --- /dev/null +++ b/src/load_generator/.gitlab-ci.yml @@ -0,0 +1,39 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build load_generator: + variables: + IMAGE_NAME: 'load_generator' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml diff --git a/src/tests/tools/load_gen/__init__.py b/src/load_generator/Config.py similarity index 100% rename from src/tests/tools/load_gen/__init__.py rename to src/load_generator/Config.py diff --git a/src/load_generator/Dockerfile b/src/load_generator/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..2e5427a34ab5f1c932d1aee378bc5c46ca70ef36 --- /dev/null +++ b/src/load_generator/Dockerfile @@ -0,0 +1,71 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/load_generator +WORKDIR /var/teraflow/load_generator +COPY src/load_generator/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ +COPY src/dlt/. dlt/ +COPY src/service/. service/ +COPY src/slice/. slice/ + +# Start the service +ENTRYPOINT ["python", "-m", "load_generator.service"] diff --git a/src/load_generator/README.md b/src/load_generator/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e6b0397bf435abb91f5e7c463da32367eba142cf --- /dev/null +++ b/src/load_generator/README.md @@ -0,0 +1,18 @@ +# Tool: Load Generator + +Simple tool to generate load in ETSI TeraFlowSDN controller with requests for creating services and slices. +The tool can be executed form command line or from WebUI interface. + +## Example (Command Line): + +Deploy TeraFlowSDN controller with your specific settings: +```(bash) +cd ~/tfs-ctrl +source my_deploy.sh +./deploy.sh +``` + +Run the tool: +```(bash) +./src/load_generator/run.sh +``` diff --git a/src/load_generator/__init__.py b/src/load_generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/load_generator/client/LoadGeneratorClient.py b/src/load_generator/client/LoadGeneratorClient.py new file mode 100644 index 0000000000000000000000000000000000000000..d7e215802bdbbb8f52085291c57fd4b4c82335bb --- /dev/null +++ b/src/load_generator/client/LoadGeneratorClient.py @@ -0,0 +1,60 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.proto.context_pb2 import Empty +from common.proto.load_generator_pb2_grpc import LoadGeneratorServiceStub +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class LoadGeneratorClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.LOAD_GENERATOR) + if not port: port = get_service_port_grpc(ServiceNameEnum.LOAD_GENERATOR) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = LoadGeneratorServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def Start(self, request : Empty) -> Empty: + LOGGER.debug('Start request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.Start(request) + LOGGER.debug('Start result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def Stop(self, request : Empty) -> Empty: + LOGGER.debug('Stop request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.Stop(request) + LOGGER.debug('Stop result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/load_generator/client/__init__.py b/src/load_generator/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/load_generator/command/__init__.py b/src/load_generator/command/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/command/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/tools/load_gen/__main__.py b/src/load_generator/command/__main__.py similarity index 80% rename from src/tests/tools/load_gen/__main__.py rename to src/load_generator/command/__main__.py index 9a5ea2b6949d1b6dd50d0a40407c6740bf266dd3..9f61fc0801017c1f917b4eafc5211e5361d33322 100644 --- a/src/tests/tools/load_gen/__main__.py +++ b/src/load_generator/command/__main__.py @@ -13,10 +13,11 @@ # limitations under the License. import logging, sys -from .Constants import RequestType -from .Parameters import Parameters -from .RequestGenerator import RequestGenerator -from .RequestScheduler import RequestScheduler +from apscheduler.schedulers.blocking import BlockingScheduler +from load_generator.load_gen.Constants import RequestType +from load_generator.load_gen.Parameters import Parameters +from load_generator.load_gen.RequestGenerator import RequestGenerator +from load_generator.load_gen.RequestScheduler import RequestScheduler logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) @@ -45,7 +46,7 @@ def main(): generator.initialize() LOGGER.info('Running Schedule...') - scheduler = RequestScheduler(parameters, generator) + scheduler = RequestScheduler(parameters, generator, scheduler_class=BlockingScheduler) scheduler.start() LOGGER.info('Done!') diff --git a/src/tests/tools/load_gen/Constants.py b/src/load_generator/load_gen/Constants.py similarity index 100% rename from src/tests/tools/load_gen/Constants.py rename to src/load_generator/load_gen/Constants.py diff --git a/src/tests/tools/load_gen/DltTools.py b/src/load_generator/load_gen/DltTools.py similarity index 100% rename from src/tests/tools/load_gen/DltTools.py rename to src/load_generator/load_gen/DltTools.py diff --git a/src/tests/tools/load_gen/Parameters.py b/src/load_generator/load_gen/Parameters.py similarity index 100% rename from src/tests/tools/load_gen/Parameters.py rename to src/load_generator/load_gen/Parameters.py diff --git a/src/tests/tools/load_gen/RequestGenerator.py b/src/load_generator/load_gen/RequestGenerator.py similarity index 99% rename from src/tests/tools/load_gen/RequestGenerator.py rename to src/load_generator/load_gen/RequestGenerator.py index d38291d380d044fa3b91a1b653ea47f6e917fe16..e983f90dc998c60d8ed95641651ed882812a1519 100644 --- a/src/tests/tools/load_gen/RequestGenerator.py +++ b/src/load_generator/load_gen/RequestGenerator.py @@ -25,8 +25,8 @@ from common.tools.object_factory.Slice import json_slice from common.tools.object_factory.Topology import json_topology_id from context.client.ContextClient import ContextClient from dlt.connector.client.DltConnectorClient import DltConnectorClient -from tests.tools.load_gen.DltTools import record_device_to_dlt, record_link_to_dlt from .Constants import ENDPOINT_COMPATIBILITY, RequestType +from .DltTools import record_device_to_dlt, record_link_to_dlt from .Parameters import Parameters LOGGER = logging.getLogger(__name__) diff --git a/src/tests/tools/load_gen/RequestScheduler.py b/src/load_generator/load_gen/RequestScheduler.py similarity index 97% rename from src/tests/tools/load_gen/RequestScheduler.py rename to src/load_generator/load_gen/RequestScheduler.py index eafb95c30032e69ab4f2f7874656b11db4f6817f..408e0125fb63b867d2a5054b74c2841d9fea368b 100644 --- a/src/tests/tools/load_gen/RequestScheduler.py +++ b/src/load_generator/load_gen/RequestScheduler.py @@ -31,8 +31,10 @@ logging.getLogger('apscheduler.scheduler').setLevel(logging.WARNING) LOGGER = logging.getLogger(__name__) class RequestScheduler: - def __init__(self, parameters : Parameters, generator : RequestGenerator) -> None: - self._scheduler = BlockingScheduler() + def __init__( + self, parameters : Parameters, generator : RequestGenerator, scheduler_class=BlockingScheduler + ) -> None: + self._scheduler = scheduler_class() self._scheduler.configure( jobstores = {'default': MemoryJobStore()}, executors = {'default': ThreadPoolExecutor(max_workers=10)}, @@ -65,6 +67,9 @@ class RequestScheduler: self._schedule_request_setup() self._scheduler.start() + def stop(self): + self._scheduler.shutdown() + def _request_setup(self) -> None: self._schedule_request_setup() diff --git a/src/load_generator/load_gen/__init__.py b/src/load_generator/load_gen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/load_gen/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/load_generator/requirements.in b/src/load_generator/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..61a0a0efbeb0d2295df8d8dacdee3f7f1235f80a --- /dev/null +++ b/src/load_generator/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +APScheduler==3.8.1 diff --git a/src/tests/tools/load_gen/run.sh b/src/load_generator/run.sh similarity index 90% rename from src/tests/tools/load_gen/run.sh rename to src/load_generator/run.sh index b16808ab6905927728212185681e2a6d4a5135ba..35db1ad4db965a7d3dfbdfe1c114c8bc0df39e2f 100755 --- a/src/tests/tools/load_gen/run.sh +++ b/src/load_generator/run.sh @@ -13,5 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Use this script to run standalone + source tfs_runtime_env_vars.sh -python -m tests.tools.load_gen +python -m load_generator.command diff --git a/src/load_generator/service/LoadGeneratorService.py b/src/load_generator/service/LoadGeneratorService.py new file mode 100644 index 0000000000000000000000000000000000000000..0127e5f86f9f46e50ec5c15c65997255f5587d88 --- /dev/null +++ b/src/load_generator/service/LoadGeneratorService.py @@ -0,0 +1,28 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.proto.load_generator_pb2_grpc import add_LoadGeneratorServiceServicer_to_server +from common.tools.service.GenericGrpcService import GenericGrpcService +from .LoadGeneratorServiceServicerImpl import LoadGeneratorServiceServicerImpl + +class LoadGeneratorService(GenericGrpcService): + def __init__(self, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.LOAD_GENERATOR) + super().__init__(port, cls_name=cls_name) + self.load_generator_servicer = LoadGeneratorServiceServicerImpl() + + def install_servicers(self): + add_LoadGeneratorServiceServicer_to_server(self.load_generator_servicer, self.server) diff --git a/src/load_generator/service/LoadGeneratorServiceServicerImpl.py b/src/load_generator/service/LoadGeneratorServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..1fa653394fb7bea268f96e4b2ca2c7d98ed40d22 --- /dev/null +++ b/src/load_generator/service/LoadGeneratorServiceServicerImpl.py @@ -0,0 +1,63 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional +import grpc, logging +from apscheduler.schedulers.background import BackgroundScheduler +from common.proto.context_pb2 import Empty +from common.proto.load_generator_pb2_grpc import LoadGeneratorServiceServicer +from load_generator.load_gen.Constants import RequestType +from load_generator.load_gen.Parameters import Parameters +from load_generator.load_gen.RequestGenerator import RequestGenerator +from load_generator.load_gen.RequestScheduler import RequestScheduler + +LOGGER = logging.getLogger(__name__) + +class LoadGeneratorServiceServicerImpl(LoadGeneratorServiceServicer): + def __init__(self): + LOGGER.debug('Creating Servicer...') + self._parameters = Parameters( + num_requests = 100, + request_types = [ + RequestType.SERVICE_L2NM, + RequestType.SERVICE_L3NM, + #RequestType.SERVICE_MW, + #RequestType.SERVICE_TAPI, + RequestType.SLICE_L2NM, + RequestType.SLICE_L3NM, + ], + offered_load = 50, + holding_time = 10, + dry_mode = False, # in dry mode, no request is sent to TeraFlowSDN + record_to_dlt = False, # if record_to_dlt, changes in device/link/service/slice are uploaded to DLT + dlt_domain_id = 'dlt-perf-eval', # domain used to uploaded entities, ignored when record_to_dlt = False + ) + self._generator : Optional[RequestGenerator] = None + self._scheduler : Optional[RequestScheduler] = None + LOGGER.debug('Servicer Created') + + def Start(self, request : Empty, context : grpc.ServicerContext) -> Empty: + LOGGER.info('Initializing Generator...') + self._generator = RequestGenerator(self._parameters) + self._generator.initialize() + + LOGGER.info('Running Schedule...') + self._scheduler = RequestScheduler(self._parameters, self._generator, scheduler_class=BackgroundScheduler) + self._scheduler.start() + return Empty() + + def Stop(self, request : Empty, context : grpc.ServicerContext) -> Empty: + if self._scheduler is not None: + self._scheduler.stop() + return Empty() diff --git a/src/load_generator/service/__init__.py b/src/load_generator/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/load_generator/service/__main__.py b/src/load_generator/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f49ee244532f6e035a35f7e5e1d02f0d9a1767d --- /dev/null +++ b/src/load_generator/service/__main__.py @@ -0,0 +1,64 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, + wait_for_environment_variables) +from .LoadGeneratorService import LoadGeneratorService + +log_level = get_log_level() +logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") +LOGGER = logging.getLogger(__name__) + +terminate = threading.Event() + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.SERVICE, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.SERVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.SLICE, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.SLICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Starting load generator service + grpc_service = LoadGeneratorService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=0.1): pass + + scheduler = grpc_service.load_generator_servicer._scheduler + if scheduler is not None: scheduler.stop() + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/load_generator/tests/__init__.py b/src/load_generator/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/load_generator/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/tools/load_gen/deploy_specs.sh b/src/load_generator/tests/deploy_specs.sh similarity index 95% rename from src/tests/tools/load_gen/deploy_specs.sh rename to src/load_generator/tests/deploy_specs.sh index a688f1c0ad920bab2fb5157dce72225671ed837e..a5af70b04a84ffa83b0e19da005175f0291c4f93 100644 --- a/src/tests/tools/load_gen/deploy_specs.sh +++ b/src/load_generator/tests/deploy_specs.sh @@ -7,7 +7,7 @@ export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/" # interdomain slice pathcomp dlt # dbscanserving opticalattackmitigator opticalattackdetector # l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector -export TFS_COMPONENTS="context device pathcomp service slice webui" # automation monitoring compute dlt +export TFS_COMPONENTS="context device pathcomp service slice webui load_generator" # automation monitoring compute dlt # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/tests/tools/load_gen/descriptors.json b/src/load_generator/tests/descriptors.json similarity index 100% rename from src/tests/tools/load_gen/descriptors.json rename to src/load_generator/tests/descriptors.json diff --git a/src/tests/tools/load_gen/test_dlt_functional.py b/src/load_generator/tests/test_dlt_functional.py similarity index 100% rename from src/tests/tools/load_gen/test_dlt_functional.py rename to src/load_generator/tests/test_dlt_functional.py diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index d60cca6597ced52db8e320f3ba1beb2b032be65b..84a43d370395523852ec3d0190826c78c2c6bffb 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -69,6 +69,9 @@ def create_app(use_config=None, web_app_root=None): from webui.service.main.routes import main app.register_blueprint(main) + from webui.service.load_gen.routes import load_gen + app.register_blueprint(load_gen) + from webui.service.service.routes import service app.register_blueprint(service) diff --git a/src/webui/service/load_gen/__init__.py b/src/webui/service/load_gen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70a33251242c51f49140e596b8208a19dd5245f7 --- /dev/null +++ b/src/webui/service/load_gen/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/webui/service/load_gen/routes.py b/src/webui/service/load_gen/routes.py new file mode 100644 index 0000000000000000000000000000000000000000..fc091f3b4a5ea732d89599154659d9cbda20629b --- /dev/null +++ b/src/webui/service/load_gen/routes.py @@ -0,0 +1,45 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import render_template, Blueprint, flash +from common.proto.context_pb2 import Empty +from load_generator.client.LoadGeneratorClient import LoadGeneratorClient + +load_gen = Blueprint('load_gen', __name__, url_prefix='/load_gen') + +@load_gen.route('start', methods=['GET']) +def start(): + load_gen_client = LoadGeneratorClient() + try: + load_gen_client.connect() + load_gen_client.Start(Empty()) + load_gen_client.close() + flash('Load Generator Started.', 'success') + except Exception as e: # pylint: disable=broad-except + flash('Problem starting Load Generator. {:s}'.format(str(e)), 'danger') + + return render_template('main/debug.html') + +@load_gen.route('stop', methods=['GET']) +def stop(): + load_gen_client = LoadGeneratorClient() + try: + load_gen_client.connect() + load_gen_client.Stop(Empty()) + load_gen_client.close() + flash('Load Generator Stoped.', 'success') + except Exception as e: # pylint: disable=broad-except + flash('Problem stopping Load Generator. {:s}'.format(str(e)), 'danger') + + return render_template('main/debug.html') diff --git a/src/webui/service/templates/main/debug.html b/src/webui/service/templates/main/debug.html index d065cc49d7262940beedd5eb9aa44a2ab890a07e..4b3e289c3004fcef5cccc4cbd9f40712a18ae83e 100644 --- a/src/webui/service/templates/main/debug.html +++ b/src/webui/service/templates/main/debug.html @@ -19,18 +19,25 @@ {% block content %} <h1>Debug</h1> - <h3>Dump ContextDB:</h3> - <ul> - <li> - <a class="nav-link" href="/context/api/dump/html" id="context_html_link" target="context_html"> - as HTML - </a> - </li> - <li> - <a class="nav-link" href="/context/api/dump/text" id="context_text_link" target="context_text"> - as Text - </a> - </li> - </ul> + <!-- + <h3>Dump ContextDB:</h3> + <ul> + <li> + <a class="nav-link" href="/context/api/dump/html" id="context_html_link" target="context_html"> + as HTML + </a> + </li> + <li> + <a class="nav-link" href="/context/api/dump/text" id="context_text_link" target="context_text"> + as Text + </a> + </li> + </ul> + --> + + <h3>Load Generator:</h3> + <a href="{{ url_for('load_gen.run') }}" class="btn btn-primary" style="margin-bottom: 10px;"> + Run + </a> {% endblock %}