From b4b217029b981702d6cccddf9fe1a22f7e4909ed Mon Sep 17 00:00:00 2001 From: mansoca <carlos.manso@cttc.es> Date: Wed, 27 Mar 2024 11:45:39 +0000 Subject: [PATCH] Creation VTNManager service --- my_deploy.sh | 3 + src/vnt_manager/.gitlab-ci.yml | 38 ++++++++ src/vnt_manager/Config.py | 13 +++ src/vnt_manager/Dockerfile | 84 ++++++++++++++++ src/vnt_manager/__init__.py | 13 +++ src/vnt_manager/client/VNTManagerClient.py | 71 ++++++++++++++ src/vnt_manager/client/__init__.py | 13 +++ src/vnt_manager/requirements.in | 15 +++ src/vnt_manager/service/VNTManagerService.py | 35 +++++++ .../service/VNTManagerServiceServicerImpl.py | 95 +++++++++++++++++++ src/vnt_manager/service/__init__.py | 13 +++ src/vnt_manager/service/__main__.py | 80 ++++++++++++++++ 12 files changed, 473 insertions(+) create mode 100644 src/vnt_manager/.gitlab-ci.yml create mode 100644 src/vnt_manager/Config.py create mode 100644 src/vnt_manager/Dockerfile create mode 100644 src/vnt_manager/__init__.py create mode 100644 src/vnt_manager/client/VNTManagerClient.py create mode 100644 src/vnt_manager/client/__init__.py create mode 100644 src/vnt_manager/requirements.in create mode 100644 src/vnt_manager/service/VNTManagerService.py create mode 100644 src/vnt_manager/service/VNTManagerServiceServicerImpl.py create mode 100644 src/vnt_manager/service/__init__.py create mode 100644 src/vnt_manager/service/__main__.py diff --git a/my_deploy.sh b/my_deploy.sh index 7dd5e5c3e..212dd7bd6 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -52,6 +52,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate E2E Orchestrator #export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +# Uncomment to activate VNT Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} vnt_manager" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/vnt_manager/.gitlab-ci.yml b/src/vnt_manager/.gitlab-ci.yml new file mode 100644 index 000000000..d1b9da495 --- /dev/null +++ b/src/vnt_manager/.gitlab-ci.yml @@ -0,0 +1,38 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build, tag and push the Docker image to the gitlab registry +build vntmanager: + variables: + IMAGE_NAME: 'vntmanager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml diff --git a/src/vnt_manager/Config.py b/src/vnt_manager/Config.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/Config.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/Dockerfile b/src/vnt_manager/Dockerfile new file mode 100644 index 000000000..8f40741ee --- /dev/null +++ b/src/vnt_manager/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Creating a user for security reasons +RUN groupadd -r teraflow && useradd -u 1001 --no-log-init -r -m -g teraflow teraflow +USER teraflow + +# set working directory +RUN mkdir -p /home/teraflow/controller/common/ +WORKDIR /home/teraflow/controller + +# Get Python packages per module +ENV VIRTUAL_ENV=/home/teraflow/venv +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +COPY --chown=teraflow:teraflow common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /home/teraflow/controller/common +COPY --chown=teraflow:teraflow src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /home/teraflow/controller/common/proto +WORKDIR /home/teraflow/controller/common/proto +RUN touch __init__.py +COPY --chown=teraflow:teraflow proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create module sub-folders +RUN mkdir -p /home/teraflow/controller/vnt_manager +WORKDIR /home/teraflow/controller + +# Get Python packages per module +COPY --chown=teraflow:teraflow ./src/vnt_manager/requirements.in vnt_manager/requirements.in +# consider common and specific requirements to avoid inconsistencies with dependencies +RUN pip-compile --quiet --output-file=vnt_manager/requirements.txt vnt_manager/requirements.in common_requirements.in +RUN python3 -m pip install -r vnt_manager/requirements.txt + +# Add component files into working directory +COPY --chown=teraflow:teraflow ./src/context/. context +COPY --chown=teraflow:teraflow ./src/vnt_manager/. vnt_manager + +# Start the service +ENTRYPOINT ["python", "-m", "vnt_manager.service"] diff --git a/src/vnt_manager/__init__.py b/src/vnt_manager/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/client/VNTManagerClient.py b/src/vnt_manager/client/VNTManagerClient.py new file mode 100644 index 000000000..95db3b6da --- /dev/null +++ b/src/vnt_manager/client/VNTManagerClient.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import grpc + +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceStub +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import delay_exponential, retry +from common.tools.grpc.Tools import grpc_message_to_json +# from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry( + max_retries=MAX_RETRIES, + delay_function=DELAY_FUNCTION, + prepare_method_name="connect", +) + + +class VNTManagerClient: + def __init__(self, host=None, port=None): + if not host: + host = get_service_host(ServiceNameEnum.VNTMANAGER) + if not port: + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + self.endpoint = "{:s}:{:s}".format(str(host), str(port)) + LOGGER.debug("Creating channel to {:s}...".format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug("Channel created") + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = VNTManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: + self.channel.close() + self.channel = None + self.stub = None + + """ + @RETRY_DECORATOR + def Compute(self, request: E2EOrchestratorRequest) -> E2EOrchestratorReply: + LOGGER.info( + "Compute request: {:s}".format(str(grpc_message_to_json(request))) + ) + response = self.stub.Compute(request) + LOGGER.info( + "Compute result: {:s}".format(str(grpc_message_to_json(response))) + ) + return response + """ \ No newline at end of file diff --git a/src/vnt_manager/client/__init__.py b/src/vnt_manager/client/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/requirements.in b/src/vnt_manager/requirements.in new file mode 100644 index 000000000..4c4720a2d --- /dev/null +++ b/src/vnt_manager/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networkx \ No newline at end of file diff --git a/src/vnt_manager/service/VNTManagerService.py b/src/vnt_manager/service/VNTManagerService.py new file mode 100644 index 000000000..b61b213a6 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerService.py @@ -0,0 +1,35 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from common.Constants import ServiceNameEnum +from common.proto.vntmanager_pb2_grpc import add_VNTManagerServiceServicer_to_server +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from .VNTManagerServiceServicerImpl import VNTManagerServiceServicerImpl + +LOGGER = logging.getLogger(__name__) + + +class VNTManagerService(GenericGrpcService): + def __init__(self, cls_name: str = __name__): + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + super().__init__(port, cls_name=cls_name) + self.vntmanager_servicer = VNTManagerServiceServicerImpl() + + def install_servicers(self): + add_VNTManagerServiceServicer_to_server( + self.vntmanager_servicer, self.server + ) diff --git a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py new file mode 100644 index 000000000..4869218a7 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py @@ -0,0 +1,95 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import networkx as nx +import grpc +import copy + +from common.Constants import ServiceNameEnum +from common.method_wrappers.Decorator import (MetricsPool, MetricTypeEnum, safe_and_metered_rpc_method) +from common.proto.vntmanager_pb2 import VNTManagerRequest, VNTManagerReply +from common.proto.context_pb2 import Empty, Connection, EndPointId +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceServicer +from context.client.ContextClient import ContextClient +from context.service.database.uuids.EndPoint import endpoint_get_uuid + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool("VNTManager", "RPC") + +context_client: ContextClient = ContextClient() + + +class E2EOrchestratorServiceServicerImpl(VNTManagerServiceServicer): + def __init__(self): + LOGGER.debug("Creating Servicer...") + LOGGER.debug("Servicer Created") + + """ + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def Compute(self, request: E2EOrchestratorRequest, context: grpc.ServicerContext) -> E2EOrchestratorReply: + endpoints_ids = [] + for endpoint_id in request.service.service_endpoint_ids: + endpoints_ids.append(endpoint_get_uuid(endpoint_id)[2]) + + graph = nx.Graph() + + devices = context_client.ListDevices(Empty()).devices + + for device in devices: + endpoints_uuids = [endpoint.endpoint_id.endpoint_uuid.uuid + for endpoint in device.device_endpoints] + for ep in endpoints_uuids: + graph.add_node(ep) + + for ep in endpoints_uuids: + for ep_i in endpoints_uuids: + if ep == ep_i: + continue + graph.add_edge(ep, ep_i) + + links = context_client.ListLinks(Empty()).links + for link in links: + eps = [] + for endpoint_id in link.link_endpoint_ids: + eps.append(endpoint_id.endpoint_uuid.uuid) + graph.add_edge(eps[0], eps[1]) + + + shortest = nx.shortest_path(graph, endpoints_ids[0], endpoints_ids[1]) + + path = E2EOrchestratorReply() + path.services.append(copy.deepcopy(request.service)) + for i in range(0, int(len(shortest)/2)): + conn = Connection() + ep_a_uuid = str(shortest[i*2]) + ep_z_uuid = str(shortest[i*2+1]) + + conn.connection_id.connection_uuid.uuid = str(ep_a_uuid) + '_->_' + str(ep_z_uuid) + + ep_a_id = EndPointId() + ep_a_id.endpoint_uuid.uuid = ep_a_uuid + conn.path_hops_endpoint_ids.append(ep_a_id) + + ep_z_id = EndPointId() + ep_z_id.endpoint_uuid.uuid = ep_z_uuid + conn.path_hops_endpoint_ids.append(ep_z_id) + + path.connections.append(conn) + + return path + """ \ No newline at end of file diff --git a/src/vnt_manager/service/__init__.py b/src/vnt_manager/service/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/service/__main__.py b/src/vnt_manager/service/__main__.py new file mode 100644 index 000000000..03fb4dd5d --- /dev/null +++ b/src/vnt_manager/service/__main__.py @@ -0,0 +1,80 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import signal +import sys +import threading + +from prometheus_client import start_http_server + +from common.Constants import ServiceNameEnum +from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, + ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, + get_log_level, get_metrics_port, + wait_for_environment_variables) + +from .VNTManagerService import VNTManagerService + +terminate = threading.Event() +LOGGER = None + + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning("Terminate signal received") + terminate.set() + + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables( + [ + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_HOST), + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ] + ) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info("Starting...") + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting CentralizedCybersecurity service + grpc_service = VNTManagerService() + grpc_service.start() + LOGGER.info("Started...") + # Wait for Ctrl+C or termination signal + + while not terminate.wait(timeout=1): + pass + + + LOGGER.info("Terminating...") + grpc_service.stop() + + LOGGER.info("Bye") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab