diff --git a/manifests/e2e_orchestratorservice.yaml b/manifests/e2e_orchestratorservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..899e17fff32e02473d7249eda937282d394efa4e --- /dev/null +++ b/manifests/e2e_orchestratorservice.yaml @@ -0,0 +1,91 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: e2e-orchestratorservice +spec: + selector: + matchLabels: + app: e2e-orchestratorservice + template: + metadata: + labels: + app: e2e-orchestratorservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/e2e_orchestrator:latest + imagePullPolicy: Always + ports: + - containerPort: 10050 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10050"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:10050"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: e2e-orchestratorservice + labels: + app: e2e-orchestratorservice +spec: + type: ClusterIP + selector: + app: e2e-orchestratorservice + ports: + - name: grpc + port: 10050 + targetPort: 10050 + - name: metrics + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: e2e-orchestratorservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: e2e-orchestratorservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/my_deploy.sh b/my_deploy.sh index a9f3f00e33d8f56f9145c7dd4069d229d48d1117..0fcb51f90509732452620126c9062597a17735ed 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -43,6 +43,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate Forecaster #export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/proto/e2eorchestrator.proto b/proto/e2eorchestrator.proto new file mode 100644 index 0000000000000000000000000000000000000000..9eed8523e52faa32d4397bb4635a1cf0c56aa4d5 --- /dev/null +++ b/proto/e2eorchestrator.proto @@ -0,0 +1,39 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// protocol buffers documentation: https://developers.google.com/protocol-buffers/docs/proto3 +syntax = "proto3"; +package orchestrator; + +import "context.proto"; + + +service E2EOrchestratorService { + rpc Compute(E2EOrchestratorRequest) returns (E2EOrchestratorReply) {} +} + +message E2EOrchestratorRequest { + context.Service service = 1; +} + +message E2EOrchestratorReply { + // Service requested completed with possible missing fields, and + // sub-services required for supporting requested service on the + // underlying layers. + repeated context.Service services = 1; + + // Connections supporting the requested service and sub-services + // required for the underlying layers. + repeated context.Connection connections = 2; +} \ No newline at end of file diff --git a/scripts/show_logs_e2eorchestrator.sh b/scripts/show_logs_e2eorchestrator.sh new file mode 100644 index 0000000000000000000000000000000000000000..84951ed8dc24c145715c971b76cd22b79a920f98 --- /dev/null +++ b/scripts/show_logs_e2eorchestrator.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/e2eorchestratorservice -c server diff --git a/src/common/Constants.py b/src/common/Constants.py index 79d5bb3b5675dfda8f31d84a935c6d16ee3791f7..30aa09b4caa34a54dd54126195b3f322c07c932e 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -58,6 +58,7 @@ class ServiceNameEnum(Enum): CACHING = 'caching' TE = 'te' FORECASTER = 'forecaster' + E2EORCHESTRATOR = 'e2eorchestrator' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -84,6 +85,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.PATHCOMP .value : 10020, ServiceNameEnum.TE .value : 10030, ServiceNameEnum.FORECASTER .value : 10040, + ServiceNameEnum.E2EORCHESTRATOR .value : 10050, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, diff --git a/src/e2e_orchestrator/.gitlab-ci.yml b/src/e2e_orchestrator/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..a14a215afc33067e923a7064b8f4617c61f4de9d --- /dev/null +++ b/src/e2e_orchestrator/.gitlab-ci.yml @@ -0,0 +1,38 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build, tag and push the Docker image to the gitlab registry +build e2eorchestrator: + variables: + IMAGE_NAME: 'e2eorchestrator' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml diff --git a/src/e2e_orchestrator/Config.py b/src/e2e_orchestrator/Config.py new file mode 100644 index 0000000000000000000000000000000000000000..38d04994fb0fa1951fb465bc127eb72659dc2eaf --- /dev/null +++ b/src/e2e_orchestrator/Config.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/e2e_orchestrator/Dockerfile b/src/e2e_orchestrator/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..85b7f1666bf9c49318da441eeb6bf6050def8db7 --- /dev/null +++ b/src/e2e_orchestrator/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Creating a user for security reasons +RUN groupadd -r teraflow && useradd -u 1001 --no-log-init -r -m -g teraflow teraflow +USER teraflow + +# set working directory +RUN mkdir -p /home/teraflow/controller/common/ +WORKDIR /home/teraflow/controller + +# Get Python packages per module +ENV VIRTUAL_ENV=/home/teraflow/venv +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +COPY --chown=teraflow:teraflow common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /home/teraflow/controller/common +COPY --chown=teraflow:teraflow src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /home/teraflow/controller/common/proto +WORKDIR /home/teraflow/controller/common/proto +RUN touch __init__.py +COPY --chown=teraflow:teraflow proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create module sub-folders +RUN mkdir -p /home/teraflow/controller/e2e_orchestrator +WORKDIR /home/teraflow/controller + +# Get Python packages per module +COPY --chown=teraflow:teraflow ./src/e2e_orchestrator/requirements.in e2e_orchestrator/requirements.in +# consider common and specific requirements to avoid inconsistencies with dependencies +RUN pip-compile --quiet --output-file=e2e_orchestrator/requirements.txt e2e_orchestrator/requirements.in common_requirements.in +RUN python3 -m pip install -r e2e_orchestrator/requirements.txt + +# Add component files into working directory +COPY --chown=teraflow:teraflow ./src/context/. context +COPY --chown=teraflow:teraflow ./src/e2e_orchestrator/. e2e_orchestrator + +# Start the service +ENTRYPOINT ["python", "-m", "e2e_orchestrator.service"] diff --git a/src/e2e_orchestrator/__init__.py b/src/e2e_orchestrator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38d04994fb0fa1951fb465bc127eb72659dc2eaf --- /dev/null +++ b/src/e2e_orchestrator/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/e2e_orchestrator/client/E2EOrchestratorClient.py b/src/e2e_orchestrator/client/E2EOrchestratorClient.py new file mode 100644 index 0000000000000000000000000000000000000000..10f183a51a64671a3cc97cc2c13d23ffd55c03fb --- /dev/null +++ b/src/e2e_orchestrator/client/E2EOrchestratorClient.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import grpc + +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.proto.e2eorchestrator_pb2_grpc import E2EOrchestratorServiceStub +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import delay_exponential, retry +from common.tools.grpc.Tools import grpc_message_to_json +from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry( + max_retries=MAX_RETRIES, + delay_function=DELAY_FUNCTION, + prepare_method_name="connect", +) + + +class E2EOrchestratorClient: + def __init__(self, host=None, port=None): + if not host: + host = get_service_host(ServiceNameEnum.E2EORCHESTRATOR) + if not port: + port = get_service_port_grpc(ServiceNameEnum.E2EORCHESTRATOR) + self.endpoint = "{:s}:{:s}".format(str(host), str(port)) + LOGGER.debug("Creating channel to {:s}...".format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug("Channel created") + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = E2EOrchestratorServiceStub(self.channel) + + def close(self): + if self.channel is not None: + self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def Compute(self, request: E2EOrchestratorRequest) -> E2EOrchestratorReply: + LOGGER.info( + "Compute request: {:s}".format(str(grpc_message_to_json(request))) + ) + response = self.stub.Compute(request) + LOGGER.info( + "Compute result: {:s}".format(str(grpc_message_to_json(response))) + ) + return response diff --git a/src/e2e_orchestrator/client/__init__.py b/src/e2e_orchestrator/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38d04994fb0fa1951fb465bc127eb72659dc2eaf --- /dev/null +++ b/src/e2e_orchestrator/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/e2e_orchestrator/requirements.in b/src/e2e_orchestrator/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..4c4720a2df4482faeda1ad99f9d383ebb5c0f848 --- /dev/null +++ b/src/e2e_orchestrator/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networkx \ No newline at end of file diff --git a/src/e2e_orchestrator/service/E2EOrchestratorService.py b/src/e2e_orchestrator/service/E2EOrchestratorService.py new file mode 100644 index 0000000000000000000000000000000000000000..4d6125d4a11210786de5cb83970743f480d4c8cf --- /dev/null +++ b/src/e2e_orchestrator/service/E2EOrchestratorService.py @@ -0,0 +1,35 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from common.Constants import ServiceNameEnum +from common.proto.e2eorchestrator_pb2_grpc import add_E2EOrchestratorServiceServicer_to_server +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from .E2EOrchestratorServiceServicerImpl import E2EOrchestratorServiceServicerImpl + +LOGGER = logging.getLogger(__name__) + + +class E2EOrchestratorService(GenericGrpcService): + def __init__(self, cls_name: str = __name__): + port = get_service_port_grpc(ServiceNameEnum.E2EORCHESTRATOR) + super().__init__(port, cls_name=cls_name) + self.e2eorchestrator_servicer = E2EOrchestratorServiceServicerImpl() + + def install_servicers(self): + add_E2EOrchestratorServiceServicer_to_server( + self.e2eorchestrator_servicer, self.server + ) diff --git a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..d233f2e17a6485d425013ab4064501bc0bad84bf --- /dev/null +++ b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py @@ -0,0 +1,93 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import networkx as nx +import grpc +import copy + +from common.Constants import ServiceNameEnum +from common.method_wrappers.Decorator import (MetricsPool, MetricTypeEnum, safe_and_metered_rpc_method) +from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply +from common.proto.context_pb2 import Empty, Connection, EndPointId +from common.proto.e2eorchestrator_pb2_grpc import E2EOrchestratorServiceServicer +from context.client.ContextClient import ContextClient +from context.service.database.uuids.EndPoint import endpoint_get_uuid + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool("E2EOrchestrator", "RPC") + +context_client: ContextClient = ContextClient() + + +class E2EOrchestratorServiceServicerImpl(E2EOrchestratorServiceServicer): + def __init__(self): + LOGGER.debug("Creating Servicer...") + LOGGER.debug("Servicer Created") + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def Compute(self, request: E2EOrchestratorRequest, context: grpc.ServicerContext) -> E2EOrchestratorReply: + endpoints_ids = [] + for endpoint_id in request.service.service_endpoint_ids: + endpoints_ids.append(endpoint_get_uuid(endpoint_id)[2]) + + graph = nx.Graph() + + devices = context_client.ListDevices(Empty()).devices + + for device in devices: + endpoints_uuids = [endpoint.endpoint_id.endpoint_uuid.uuid + for endpoint in device.device_endpoints] + for ep in endpoints_uuids: + graph.add_node(ep) + + for ep in endpoints_uuids: + for ep_i in endpoints_uuids: + if ep == ep_i: + continue + graph.add_edge(ep, ep_i) + + links = context_client.ListLinks(Empty()).links + for link in links: + eps = [] + for endpoint_id in link.link_endpoint_ids: + eps.append(endpoint_id.endpoint_uuid.uuid) + graph.add_edge(eps[0], eps[1]) + + + shortest = nx.shortest_path(graph, endpoints_ids[0], endpoints_ids[1]) + + path = E2EOrchestratorReply() + path.services.append(copy.deepcopy(request.service)) + for i in range(0, int(len(shortest)/2)): + conn = Connection() + ep_a_uuid = str(shortest[i*2]) + ep_z_uuid = str(shortest[i*2+1]) + + conn.connection_id.connection_uuid.uuid = str(ep_a_uuid) + '_->_' + str(ep_z_uuid) + + ep_a_id = EndPointId() + ep_a_id.endpoint_uuid.uuid = ep_a_uuid + conn.path_hops_endpoint_ids.append(ep_a_id) + + ep_z_id = EndPointId() + ep_z_id.endpoint_uuid.uuid = ep_z_uuid + conn.path_hops_endpoint_ids.append(ep_z_id) + + path.connections.append(conn) + + return path diff --git a/src/e2e_orchestrator/service/__init__.py b/src/e2e_orchestrator/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38d04994fb0fa1951fb465bc127eb72659dc2eaf --- /dev/null +++ b/src/e2e_orchestrator/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/e2e_orchestrator/service/__main__.py b/src/e2e_orchestrator/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..a586543a7078d9b7f868967ad7eea7d228985086 --- /dev/null +++ b/src/e2e_orchestrator/service/__main__.py @@ -0,0 +1,80 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import signal +import sys +import threading + +from prometheus_client import start_http_server + +from common.Constants import ServiceNameEnum +from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, + ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, + get_log_level, get_metrics_port, + wait_for_environment_variables) + +from .E2EOrchestratorService import E2EOrchestratorService + +terminate = threading.Event() +LOGGER = None + + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning("Terminate signal received") + terminate.set() + + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables( + [ + get_env_var_name(ServiceNameEnum.E2EORCHESTRATOR, ENVVAR_SUFIX_SERVICE_HOST), + get_env_var_name(ServiceNameEnum.E2EORCHESTRATOR, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ] + ) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info("Starting...") + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting CentralizedCybersecurity service + grpc_service = E2EOrchestratorService() + grpc_service.start() + LOGGER.info("Started...") + # Wait for Ctrl+C or termination signal + + while not terminate.wait(timeout=1): + pass + + + LOGGER.info("Terminating...") + grpc_service.stop() + + LOGGER.info("Bye") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/e2e_orchestrator/tests/__init__.py b/src/e2e_orchestrator/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1549d9811aa5d1c193a44ad45d0d7773236c0612 --- /dev/null +++ b/src/e2e_orchestrator/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/e2e_orchestrator/tests/deploy_specs.sh b/src/e2e_orchestrator/tests/deploy_specs.sh new file mode 100755 index 0000000000000000000000000000000000000000..67aed976a4f89d13e28470491a6260d5d5ea3a35 --- /dev/null +++ b/src/e2e_orchestrator/tests/deploy_specs.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" + +# Uncomment to activate Monitoring +# export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate ZTP and Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp policy" +# export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate E2E_Orchestrator +export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + + + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/src/e2e_orchestrator/tests/descriptors_emulated.json b/src/e2e_orchestrator/tests/descriptors_emulated.json new file mode 100644 index 0000000000000000000000000000000000000000..a2918ace990550cac63d5eb7cd0cf69bf5da5538 --- /dev/null +++ b/src/e2e_orchestrator/tests/descriptors_emulated.json @@ -0,0 +1,250 @@ +{ + "contexts": [ + { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_ids": [], "service_ids": [] + } + ], + "topologies": [ + { + "topology_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_uuid": {"uuid": "admin"} + }, + "device_ids": [ + {"device_uuid": {"uuid": "R1"}}, + {"device_uuid": {"uuid": "R2"}}, + {"device_uuid": {"uuid": "T1"}}, + {"device_uuid": {"uuid": "T2"}}, + {"device_uuid": {"uuid": "M1"}}, + {"device_uuid": {"uuid": "M2"}} + ], + "link_ids": [ + {"link_uuid": {"uuid": "R1==T1"}}, + {"link_uuid": {"uuid": "T1==R1"}}, + {"link_uuid": {"uuid": "R2==T2"}}, + {"link_uuid": {"uuid": "T2==R2"}}, + + {"link_uuid": {"uuid": "T1==M1"}}, + {"link_uuid": {"uuid": "M1==T1"}}, + {"link_uuid": {"uuid": "T2==M2"}}, + {"link_uuid": {"uuid": "M2==T2"}}, + + {"link_uuid": {"uuid": "M1==M2"}}, + {"link_uuid": {"uuid": "M2==M1"}} + + + ] + } + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "R1"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R2"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "T1"}}, "device_type": "emu-optical-transponder", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "T2"}}, "device_type": "emu-optical-transponder", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + }, + + { + "device_id": {"device_uuid": {"uuid": "M1"}}, "device_type": "emu-optical-roadm", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + }, + + + { + "device_id": {"device_uuid": {"uuid": "M2"}}, "device_type": "emu-optical-roadm", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "3/1"}, + {"sample_types": [], "type": "copper", "uuid": "3/2"}, + {"sample_types": [], "type": "copper", "uuid": "3/3"}, + {"sample_types": [], "type": "copper", "uuid": "3/4"} + ]}}} + ]} + } + + + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "R1==T1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "1/1"}}, + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "T1==R1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "1/1"}}, + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R2==T2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "1/2"}}, + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "T2==R2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "1/2"}}, + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "1/2"}} + ] + }, + + + { + "link_id": {"link_uuid": {"uuid": "T1==M1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "M1"}}, "endpoint_uuid": {"uuid": "2/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "M1==T1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "M1"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "T1"}}, "endpoint_uuid": {"uuid": "2/1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "T2==M2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "2/2"}}, + {"device_id": {"device_uuid": {"uuid": "M2"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "M2==T2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "M2"}}, "endpoint_uuid": {"uuid": "2/2"}}, + {"device_id": {"device_uuid": {"uuid": "T2"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "M1==M2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "M1"}}, "endpoint_uuid": {"uuid": "3/1"}}, + {"device_id": {"device_uuid": {"uuid": "M2"}}, "endpoint_uuid": {"uuid": "3/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "M2==M1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "M2"}}, "endpoint_uuid": {"uuid": "3/1"}}, + {"device_id": {"device_uuid": {"uuid": "M1"}}, "endpoint_uuid": {"uuid": "3/1"}} + ] + } + + ] +} \ No newline at end of file diff --git a/src/e2e_orchestrator/tests/functional_tests/Fixtures.py b/src/e2e_orchestrator/tests/functional_tests/Fixtures.py new file mode 100644 index 0000000000000000000000000000000000000000..0f7b4b0d428768fa35d77a8c30ed5fc3e1a2b8d2 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/Fixtures.py @@ -0,0 +1,39 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from monitoring.client.MonitoringClient import MonitoringClient +from e2e_orchestrator.client.E2EOrchestratorClient import E2EOrchestratorClient +from service.client.ServiceClient import ServiceClient + + +@pytest.fixture(scope='session') +def context_client(): + _client = ContextClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def device_client(): + _client = DeviceClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def e2eorchestrator_client(): + _client = E2EOrchestratorClient() + yield _client + _client.close() diff --git a/src/e2e_orchestrator/tests/functional_tests/Objects.py b/src/e2e_orchestrator/tests/functional_tests/Objects.py new file mode 100644 index 0000000000000000000000000000000000000000..1748efec9325e0651ee7fc8dcfa62454aa29e385 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/Objects.py @@ -0,0 +1,60 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, List, Tuple +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME +from common.tools.object_factory.Context import json_context, json_context_id +from common.tools.object_factory.Device import ( + json_device_connect_rules, json_device_emulated_connect_rules, json_device_emulated_packet_router_disabled, + json_device_connect_rules, json_device_id, json_device_p4_disabled, + json_device_emulated_tapi_disabled, json_device_id, json_device_packetrouter_disabled, json_device_tapi_disabled) +from common.tools.object_factory.Service import ( + get_service_uuid, json_service_l3nm_planned,json_service_p4_planned) +from common.tools.object_factory.ConfigRule import ( + json_config_rule_set, json_config_rule_delete) +from common.tools.object_factory.EndPoint import json_endpoint, json_endpoint_ids, json_endpoints, json_endpoint_id +from common.tools.object_factory.EndPoint import json_endpoint_descriptor + + + +DEVICE_R1_UUID = 'R1' +DEVICE_R2_UUID = 'R2' + +DEVICE_R1_ID = json_device_id(DEVICE_R1_UUID) +DEVICE_R1_ENDPOINT_DEFS = [json_endpoint_descriptor('2/2', 'port')] +DEVICE_R2_ID = json_device_id(DEVICE_R2_UUID) +DEVICE_R2_ENDPOINT_DEFS = [json_endpoint_descriptor('2/2', 'port')] + +DEVICE_R1_ENDPOINTS = json_endpoints(DEVICE_R1_ID, DEVICE_R1_ENDPOINT_DEFS) +DEVICE_R2_ENDPOINTS = json_endpoints(DEVICE_R2_ID, DEVICE_R2_ENDPOINT_DEFS) + + +DEVICE_R1_ENDPOINT_IDS = json_endpoint_ids(DEVICE_R1_ID, DEVICE_R1_ENDPOINT_DEFS) +ENDPOINT_ID_R1 = DEVICE_R1_ENDPOINTS[0]['endpoint_id'] +DEVICE_R2_ENDPOINT_IDS = json_endpoint_ids(DEVICE_R2_ID, DEVICE_R2_ENDPOINT_DEFS) +ENDPOINT_ID_R2 = DEVICE_R2_ENDPOINTS[0]['endpoint_id'] + + +# ----- Service ---------------------------------------------------------------------------------------------------------- + + +SERVICE_R1_R2_UUID = get_service_uuid(ENDPOINT_ID_R1, ENDPOINT_ID_R2) +SERVICE_R1_R2 = json_service_p4_planned(SERVICE_R1_R2_UUID) +SERVICE_R1_R2_ENDPOINT_IDS = [DEVICE_R1_ENDPOINT_IDS[0], DEVICE_R2_ENDPOINT_IDS[0]] + + +SERVICES = [ + (SERVICE_R1_R2, SERVICE_R1_R2_ENDPOINT_IDS) +] diff --git a/src/e2e_orchestrator/tests/functional_tests/__init__.py b/src/e2e_orchestrator/tests/functional_tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1549d9811aa5d1c193a44ad45d0d7773236c0612 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/e2e_orchestrator/tests/functional_tests/test_functional_bootstrap.py b/src/e2e_orchestrator/tests/functional_tests/test_functional_bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..d0e85e4e33c2253e5def8b093b84d0e027eee3f3 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/test_functional_bootstrap.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.proto.monitoring_pb2 import KpiDescriptorList +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from monitoring.client.MonitoringClient import MonitoringClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = 'src/e2e_orchestrator/tests/descriptors_emulated.json' +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_bootstrap( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 1): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 + assert num_devices_enabled == num_devices + + diff --git a/src/e2e_orchestrator/tests/functional_tests/test_functional_cleanup.py b/src/e2e_orchestrator/tests/functional_tests/test_functional_cleanup.py new file mode 100644 index 0000000000000000000000000000000000000000..9c88b3b3886b6208062d0cae5d7e40eba1b89dc6 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/test_functional_cleanup.py @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.tools.descriptor.Loader import DescriptorLoader, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = 'src/e2e_orchestrator/tests/descriptors_emulated.json' +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_cleanup( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) diff --git a/src/e2e_orchestrator/tests/functional_tests/test_functional_compute_path.py b/src/e2e_orchestrator/tests/functional_tests/test_functional_compute_path.py new file mode 100644 index 0000000000000000000000000000000000000000..91a928e6ab82cf096479b1dbec3d203721372136 --- /dev/null +++ b/src/e2e_orchestrator/tests/functional_tests/test_functional_compute_path.py @@ -0,0 +1,61 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, random +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, Empty, ServiceTypeEnum, Service +from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.descriptor.Loader import DescriptorLoader +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from service.client.ServiceClient import ServiceClient +from .Fixtures import context_client, device_client, e2eorchestrator_client # pylint: disable=unused-import +from e2e_orchestrator.client.E2EOrchestratorClient import E2EOrchestratorClient +from .Objects import SERVICES +import copy + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = 'src/e2e_orchestrator/tests/descriptors_emulated.json' +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +def test_orchestration(context_client : ContextClient, e2eorchestrator_client : E2EOrchestratorClient): # pylint: disable=redefined-outer-name + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader(descriptors_file=DESCRIPTOR_FILE, context_client=context_client) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + + + # ----- Compute E2E path --------------------------------------------------------------- + for service, endpoints in SERVICES: + service_uuid = service['service_id']['service_uuid']['uuid'] + print('Creating Service {:s}'.format(service_uuid)) + service_p4 = copy.deepcopy(service) + service_p4['service_endpoint_ids'].extend(endpoints) + + request = E2EOrchestratorRequest() + request.service.MergeFrom(Service(**service_p4)) + reply = e2eorchestrator_client.Compute(request) + LOGGER.info(reply) + assert len(reply.connections) == 6 + diff --git a/src/e2e_orchestrator/tests/redeploy.sh b/src/e2e_orchestrator/tests/redeploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..bd995709b8065cd2860e003197ad18cd2b93cdc8 --- /dev/null +++ b/src/e2e_orchestrator/tests/redeploy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source src/e2e_orchestrator/tests/deploy_specs.sh +./deploy/all.sh +source tfs_runtime_env_vars.sh diff --git a/src/e2e_orchestrator/tests/run_test_01_bootstrap.sh b/src/e2e_orchestrator/tests/run_test_01_bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..e875cf9aee09ed5818eb6a5c004ec77cb0ed5eec --- /dev/null +++ b/src/e2e_orchestrator/tests/run_test_01_bootstrap.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source tfs_runtime_env_vars.sh +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_bootstrap.py diff --git a/src/e2e_orchestrator/tests/run_test_02_compute_path.sh b/src/e2e_orchestrator/tests/run_test_02_compute_path.sh new file mode 100755 index 0000000000000000000000000000000000000000..be61e70e44e0a11e76cb6afb787c8e1c85ed4e09 --- /dev/null +++ b/src/e2e_orchestrator/tests/run_test_02_compute_path.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source tfs_runtime_env_vars.sh +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_compute_path.py diff --git a/src/e2e_orchestrator/tests/run_test_03_cleanup.sh b/src/e2e_orchestrator/tests/run_test_03_cleanup.sh new file mode 100755 index 0000000000000000000000000000000000000000..78ed3e1d164aa0e45db77c366b2787e0a13836be --- /dev/null +++ b/src/e2e_orchestrator/tests/run_test_03_cleanup.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source tfs_runtime_env_vars.sh +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_cleanup.py diff --git a/src/e2e_orchestrator/tests/run_tests.sh b/src/e2e_orchestrator/tests/run_tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..9f19bb43801f6354e5c670374c3669ecf9a2bea6 --- /dev/null +++ b/src/e2e_orchestrator/tests/run_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run functional tests +source tfs_runtime_env_vars.sh +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_bootstrap.py +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_compute_path.py +pytest --verbose --log-level=INFO src/e2e_orchestrator/tests/functional_tests/test_functional_cleanup.py diff --git a/src/tests/Fixtures.py b/src/tests/Fixtures.py index ecb44a7588d62e14f4295141adb3acc7bbe89532..62ad13f49f0bc73040c350040fac79e89a2b6070 100644 --- a/src/tests/Fixtures.py +++ b/src/tests/Fixtures.py @@ -16,6 +16,15 @@ import pytest from context.client.ContextClient import ContextClient from device.client.DeviceClient import DeviceClient from monitoring.client.MonitoringClient import MonitoringClient +from e2e_orchestrator.client.E2EOrchestratorClient import E2EOrchestratorClient +from service.client.ServiceClient import ServiceClient + + +@pytest.fixture(scope='session') +def service_client(): + _client = ServiceClient() + yield _client + _client.close() @pytest.fixture(scope='session') def context_client(): @@ -34,3 +43,9 @@ def monitoring_client(): _client = MonitoringClient() yield _client _client.close() + +@pytest.fixture(scope='session') +def e2eorchestrator_client(): + _client = E2EOrchestratorClient() + yield _client + _client.close()