From 3fcbe99171f035cd28081da4582522a940dc1c01 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 23 Sep 2024 10:32:45 +0000 Subject: [PATCH 01/15] Modified gitlab-ci.yml to autoexecute the qkd test creating at the beginning the mocked qkd nodes --- src/device/.gitlab-ci.yml | 27 +++++++++++++++++++ .../qkd/qkd_service_handler.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 9106c96a8..402a5a1f8 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -38,6 +38,30 @@ build device: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml +# Start Mock QKD Nodes before unit testing +start_mock_nodes: + stage: deploy + script: + - bash src/tests/tools/mock_qkd_nodes/start.sh & + - sleep 10 # wait for nodes to spin up + artifacts: + paths: + - mock_nodes.log + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + +# Prepare Scenario (Start NBI, mock services) +prepare_scenario: + stage: deploy + script: + - pytest src/tests/qkd/unit/PrepareScenario.py + needs: + - start_mock_nodes + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + # Apply unit test to the component unit_test device: variables: @@ -46,6 +70,8 @@ unit_test device: stage: unit_test needs: - build device + - start_mock_nodes + - prepare_scenario before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - > @@ -68,6 +94,7 @@ unit_test device: - docker logs $IMAGE_NAME - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: diff --git a/src/service/service/service_handlers/qkd/qkd_service_handler.py b/src/service/service/service_handlers/qkd/qkd_service_handler.py index 76c67867e..097738800 100644 --- a/src/service/service/service_handlers/qkd/qkd_service_handler.py +++ b/src/service/service/service_handlers/qkd/qkd_service_handler.py @@ -17,7 +17,7 @@ import json, logging, uuid from typing import Any, Dict, List, Optional, Tuple, Union from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service -from common.proto.app_pb2 import App, QKDAppStatusEnum, QKDAppTypesEnum +from common.proto.qkd_app_pb2 import App, QKDAppStatusEnum, QKDAppTypesEnum from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type -- GitLab From 2eb3e123d604c24a295e6b5dfff2d2419eabde2f Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 26 Sep 2024 11:26:33 +0000 Subject: [PATCH 02/15] Added descriptor QKD --- .../tests/qkd/unit/descriptorQKD_links.json | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/device/tests/qkd/unit/descriptorQKD_links.json diff --git a/src/device/tests/qkd/unit/descriptorQKD_links.json b/src/device/tests/qkd/unit/descriptorQKD_links.json new file mode 100644 index 000000000..f09aaa444 --- /dev/null +++ b/src/device/tests/qkd/unit/descriptorQKD_links.json @@ -0,0 +1,77 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"topology_uuid": {"uuid": "admin"}, "context_id": {"context_uuid": {"uuid": "admin"}}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", + "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "scheme": "http" + }}} + ]} + + }, + { + "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", + "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "scheme": "http" + }}} + ]} + + }, + { + "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", + "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "scheme": "http" + }}} + ]} + + } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "QKD1/10.211.36.220:1001==QKD2/10.211.36.220:2001"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.211.36.220:1001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2001"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "QKD2/10.211.36.220:2001==QKD1/10.211.36.220:1001"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.211.36.220:1001"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "QKD2/10.211.36.220:2002==QKD3/10.211.36.220:3001"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2002"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.211.36.220:3001"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "QKD3/10.211.36.220:3001==QKD2/10.211.36.220:2002"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.211.36.220:3001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2002"}} + ] + } + + ] +} -- GitLab From 925a09c6ef1ebfe5087d7feb1903511fb5e15542 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:48:29 +0000 Subject: [PATCH 03/15] Added Docker Image prune after deploy --- deploy/tfs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 189ae11e1..da078a4f3 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -669,6 +669,10 @@ if [[ "$TFS_COMPONENTS" == *"monitoring"* ]] && [[ "$TFS_COMPONENTS" == *"webui" printf "\n\n" fi +echo "Pruning Docker Images..." +docker image prune --force +printf "\n\n" + if [ "$DOCKER_BUILD" == "docker buildx build" ]; then echo "Pruning Docker Buildx Cache..." docker buildx prune --force -- GitLab From fddb0a6343c361ef0a5121f0c6e00b4451d533bb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:48:48 +0000 Subject: [PATCH 04/15] Fix NGINX ingress controller config --- manifests/nginx_ingress_http.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/nginx_ingress_http.yaml b/manifests/nginx_ingress_http.yaml index c0a0875e2..4ee386557 100644 --- a/manifests/nginx_ingress_http.yaml +++ b/manifests/nginx_ingress_http.yaml @@ -62,8 +62,8 @@ spec: name: nbiservice port: number: 8080 - - path: /()(qkd_app/.*) - - pathType: Prefix + - path: /()(qkd_app/.*) + pathType: Prefix backend: service: name: qkdappservice -- GitLab From aee38a5f61fe0c62562e24218a624e492466f0af Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:48:56 +0000 Subject: [PATCH 05/15] Fix WebUI Dockerfile --- src/webui/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webui/Dockerfile b/src/webui/Dockerfile index 204b6f731..829508766 100644 --- a/src/webui/Dockerfile +++ b/src/webui/Dockerfile @@ -84,11 +84,11 @@ COPY --chown=webui:webui src/service/__init__.py service/__init__.py COPY --chown=webui:webui src/service/client/. service/client/ COPY --chown=webui:webui src/slice/__init__.py slice/__init__.py COPY --chown=webui:webui src/slice/client/. slice/client/ -COPY --chown=webui:webui src/app/__init__.py app/__init__.py -COPY --chown=webui:webui src/app/client/. app/client/ -COPY --chown=webui:webui src/webui/. webui/ +COPY --chown=webui:webui src/qkd_app/__init__.py qkd_app/__init__.py +COPY --chown=webui:webui src/qkd_app/client/. qkd_app/client/ COPY --chown=webui:webui src/bgpls_speaker/__init__.py bgpls_speaker/__init__.py COPY --chown=webui:webui src/bgpls_speaker/client/. bgpls_speaker/client/ +COPY --chown=webui:webui src/webui/. webui/ # Start the service ENTRYPOINT ["python", "-m", "webui.service"] -- GitLab From b6eb722502cfed9e190eebbb9c91ea70233ee812 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:53:05 +0000 Subject: [PATCH 06/15] Fix NGINX ingress controller config and qkd-app manifest --- manifests/nginx_ingress_http.yaml | 2 +- .../{qkdappservice.yaml => qkd_appservice.yaml} | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) rename manifests/{qkdappservice.yaml => qkd_appservice.yaml} (93%) diff --git a/manifests/nginx_ingress_http.yaml b/manifests/nginx_ingress_http.yaml index 4ee386557..ed713bf29 100644 --- a/manifests/nginx_ingress_http.yaml +++ b/manifests/nginx_ingress_http.yaml @@ -66,6 +66,6 @@ spec: pathType: Prefix backend: service: - name: qkdappservice + name: qkd-appservice port: number: 8005 diff --git a/manifests/qkdappservice.yaml b/manifests/qkd_appservice.yaml similarity index 93% rename from manifests/qkdappservice.yaml rename to manifests/qkd_appservice.yaml index 13f16f4e5..18f4ef996 100644 --- a/manifests/qkdappservice.yaml +++ b/manifests/qkd_appservice.yaml @@ -15,16 +15,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: qkdappservice + name: qkd-appservice spec: selector: matchLabels: - app: qkdappservice + app: qkd-appservice #replicas: 1 template: metadata: labels: - app: qkdappservice + app: qkd-appservice spec: terminationGracePeriodSeconds: 5 containers: @@ -64,13 +64,13 @@ spec: apiVersion: v1 kind: Service metadata: - name: qkdappservice + name: qkd-appservice labels: - app: qkdappservice + app: qkd-appservice spec: type: ClusterIP selector: - app: qkdappservice + app: qkd-appservice ports: - name: grpc protocol: TCP -- GitLab From dbfc72813fb743dbce8f57351fe1b3791220bcf6 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:57:26 +0000 Subject: [PATCH 07/15] Fix ServiceNameEnum.APP to ServiceNameEnum.QKD_APP --- src/qkd_app/client/QKDAppClient.py | 4 ++-- src/qkd_app/service/QKDAppService.py | 2 +- src/qkd_app/service/rest_server/RestServer.py | 4 ++-- src/webui/service/__main__.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/qkd_app/client/QKDAppClient.py b/src/qkd_app/client/QKDAppClient.py index b8c09329f..1a174df6a 100644 --- a/src/qkd_app/client/QKDAppClient.py +++ b/src/qkd_app/client/QKDAppClient.py @@ -28,8 +28,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class QKDAppClient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.APP) - if not port: port = get_service_port_grpc(ServiceNameEnum.APP) + if not host: host = get_service_host(ServiceNameEnum.QKD_APP) + if not port: port = get_service_port_grpc(ServiceNameEnum.QKD_APP) self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(self.endpoint)) self.channel = None diff --git a/src/qkd_app/service/QKDAppService.py b/src/qkd_app/service/QKDAppService.py index fcf63efba..a6c93cd81 100644 --- a/src/qkd_app/service/QKDAppService.py +++ b/src/qkd_app/service/QKDAppService.py @@ -29,7 +29,7 @@ class AppService(GenericGrpcService): def __init__( self, db_engine : sqlalchemy.engine.Engine, messagebroker : MessageBroker, cls_name: str = __name__ ) -> None: - port = get_service_port_grpc(ServiceNameEnum.APP) + port = get_service_port_grpc(ServiceNameEnum.QKD_APP) super().__init__(port, max_workers=GRPC_MAX_WORKERS, cls_name=cls_name) self.app_servicer = AppServiceServicerImpl(db_engine, messagebroker) diff --git a/src/qkd_app/service/rest_server/RestServer.py b/src/qkd_app/service/rest_server/RestServer.py index 4302a0562..e21531c5b 100644 --- a/src/qkd_app/service/rest_server/RestServer.py +++ b/src/qkd_app/service/rest_server/RestServer.py @@ -18,6 +18,6 @@ from common.tools.service.GenericRestServer import GenericRestServer class RestServer(GenericRestServer): def __init__(self, cls_name: str = __name__) -> None: - bind_port = get_service_port_http(ServiceNameEnum.APP) - base_url = get_service_baseurl_http(ServiceNameEnum.APP) + bind_port = get_service_port_http(ServiceNameEnum.QKD_APP) + base_url = get_service_baseurl_http(ServiceNameEnum.QKD_APP) super().__init__(bind_port, base_url, cls_name=cls_name) diff --git a/src/webui/service/__main__.py b/src/webui/service/__main__.py index 8cb577d71..8ec8dcb64 100644 --- a/src/webui/service/__main__.py +++ b/src/webui/service/__main__.py @@ -43,8 +43,8 @@ def main(): get_env_var_name(ServiceNameEnum.SERVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), get_env_var_name(ServiceNameEnum.SLICE, ENVVAR_SUFIX_SERVICE_HOST ), get_env_var_name(ServiceNameEnum.SLICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), - get_env_var_name(ServiceNameEnum.APP, ENVVAR_SUFIX_SERVICE_HOST ), - get_env_var_name(ServiceNameEnum.APP, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.QKD_APP, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.QKD_APP, ENVVAR_SUFIX_SERVICE_PORT_GRPC), ]) logger.info('Starting...') -- GitLab From 21d5fe973a7bb4fae4c40b1ada0ff7c6176e0f8f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 12:02:50 +0000 Subject: [PATCH 08/15] Fix WebUI imports related to qkd_app --- src/webui/service/__init__.py | 20 ++++++++++---------- src/webui/service/qkd_app/routes.py | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index 990792069..f269408e4 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -83,32 +83,32 @@ def create_app(use_config=None, web_app_root=None): app.register_blueprint(healthz, url_prefix='/healthz') - from webui.service.js.routes import js # pylint: disable=import-outside-toplevel + from webui.service.js.routes import js # pylint: disable=import-outside-toplevel app.register_blueprint(js) - from webui.service.main.routes import main # pylint: disable=import-outside-toplevel + from webui.service.main.routes import main # pylint: disable=import-outside-toplevel app.register_blueprint(main) - from webui.service.load_gen.routes import load_gen # pylint: disable=import-outside-toplevel + from webui.service.load_gen.routes import load_gen # pylint: disable=import-outside-toplevel app.register_blueprint(load_gen) - from webui.service.service.routes import service # pylint: disable=import-outside-toplevel + from webui.service.service.routes import service # pylint: disable=import-outside-toplevel app.register_blueprint(service) - from webui.service.slice.routes import slice # pylint: disable=import-outside-toplevel,redefined-builtin + from webui.service.slice.routes import slice # pylint: disable=import-outside-toplevel,redefined-builtin app.register_blueprint(slice) - from webui.service.device.routes import device # pylint: disable=import-outside-toplevel + from webui.service.device.routes import device # pylint: disable=import-outside-toplevel app.register_blueprint(device) - from webui.service.bgpls.routes import bgpls # pylint: disable=import-outside-toplevel + from webui.service.bgpls.routes import bgpls # pylint: disable=import-outside-toplevel app.register_blueprint(bgpls) - from webui.service.link.routes import link # pylint: disable=import-outside-toplevel + from webui.service.link.routes import link # pylint: disable=import-outside-toplevel app.register_blueprint(link) - from webui.service.qkd_app.routes import qkd_app as _qkd_app # pylint: disable=import-outside-toplevel - app.register_blueprint(_qkd_app) + from webui.service.qkd_app.routes import qkd_app # pylint: disable=import-outside-toplevel + app.register_blueprint(qkd_app) from webui.service.policy_rule.routes import policy_rule # pylint: disable=import-outside-toplevel app.register_blueprint(policy_rule) diff --git a/src/webui/service/qkd_app/routes.py b/src/webui/service/qkd_app/routes.py index 5cbe715dc..71243fb75 100644 --- a/src/webui/service/qkd_app/routes.py +++ b/src/webui/service/qkd_app/routes.py @@ -25,12 +25,12 @@ from qkd_app.client.QKDAppClient import QKDAppClient LOGGER = logging.getLogger(__name__) -app = Blueprint('qkd_app', __name__, url_prefix='/qkd_app') +qkd_app = Blueprint('qkd_app', __name__, url_prefix='/qkd_app') qkd_app_client = QKDAppClient() context_client = ContextClient() -@app.get('/') +@qkd_app.get('/') def home(): if 'context_uuid' not in session or 'topology_uuid' not in session: flash("Please select a context!", "warning") @@ -68,10 +68,10 @@ def home(): context_client.close() return render_template( - 'app/home.html', apps=apps, device_names=device_names, ate=QKDAppTypesEnum, ase=QKDAppStatusEnum) + 'qkd_app/home.html', apps=apps, device_names=device_names, ate=QKDAppTypesEnum, ase=QKDAppStatusEnum) -@app.route('detail/', methods=('GET', 'POST')) +@qkd_app.route('detail/', methods=('GET', 'POST')) def detail(app_uuid: str): ''' context_client.connect() @@ -87,7 +87,7 @@ def detail(app_uuid: str): ''' pass -@app.get('/delete') +@qkd_app.get('/delete') def delete(app_uuid): ''' try: -- GitLab From c0c8c6b6477d51f07613ea53ed76e699d6167c95 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:17:39 +0000 Subject: [PATCH 09/15] Corrected QKD App manifest --- manifests/qkd_appservice.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/manifests/qkd_appservice.yaml b/manifests/qkd_appservice.yaml index 18f4ef996..4f89d6c6f 100644 --- a/manifests/qkd_appservice.yaml +++ b/manifests/qkd_appservice.yaml @@ -36,10 +36,8 @@ spec: - containerPort: 9192 - containerPort: 8005 env: - - name: MB_BACKEND - value: "nats" - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: CRDB_DATABASE_APP value: "qkd_app" envFrom: -- GitLab From 573844e13934b45c834ebffd87ee8bbdc8a4531c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:17:48 +0000 Subject: [PATCH 10/15] Add script show logs QKD App --- scripts/show_logs_qkd_app.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/show_logs_qkd_app.sh diff --git a/scripts/show_logs_qkd_app.sh b/scripts/show_logs_qkd_app.sh new file mode 100755 index 000000000..f4f9ee6a1 --- /dev/null +++ b/scripts/show_logs_qkd_app.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qkd-appservice -c server -- GitLab From 31fd9c06d7db2373af9bac608ddbd4043d0027b4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:18:00 +0000 Subject: [PATCH 11/15] Add QKD tests to device --- .../tests/qkd/unit/descriptorQKD_links.json | 30 +++++++++---------- .../qkd/unit/test_application_deployment.py | 4 ++- src/device/tests/qkd/unit/test_create_apps.py | 26 ++++++++++++++++ 3 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 src/device/tests/qkd/unit/test_create_apps.py diff --git a/src/device/tests/qkd/unit/descriptorQKD_links.json b/src/device/tests/qkd/unit/descriptorQKD_links.json index f09aaa444..28a9e7d5a 100644 --- a/src/device/tests/qkd/unit/descriptorQKD_links.json +++ b/src/device/tests/qkd/unit/descriptorQKD_links.json @@ -10,7 +10,7 @@ "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" @@ -22,7 +22,7 @@ "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" @@ -34,7 +34,7 @@ "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.211.36.220"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" @@ -45,31 +45,31 @@ ], "links": [ { - "link_id": {"link_uuid": {"uuid": "QKD1/10.211.36.220:1001==QKD2/10.211.36.220:2001"}}, + "link_id": {"link_uuid": {"uuid": "QKD1/10.0.2.10:1001==QKD2/10.0.2.10:2001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.211.36.220:1001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2001"}} + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}} ] }, { - "link_id": {"link_uuid": {"uuid": "QKD2/10.211.36.220:2001==QKD1/10.211.36.220:1001"}}, + "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2001==QKD1/10.0.2.10:1001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.211.36.220:1001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}} ] }, { - "link_id": {"link_uuid": {"uuid": "QKD2/10.211.36.220:2002==QKD3/10.211.36.220:3001"}}, + "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2002==QKD3/10.0.2.10:3001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2002"}}, - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.211.36.220:3001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}} ] }, { - "link_id": {"link_uuid": {"uuid": "QKD3/10.211.36.220:3001==QKD2/10.211.36.220:2002"}}, + "link_id": {"link_uuid": {"uuid": "QKD3/10.0.2.10:3001==QKD2/10.0.2.10:2002"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.211.36.220:3001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.211.36.220:2002"}} + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}} ] } diff --git a/src/device/tests/qkd/unit/test_application_deployment.py b/src/device/tests/qkd/unit/test_application_deployment.py index 92e16663b..d10ddc523 100644 --- a/src/device/tests/qkd/unit/test_application_deployment.py +++ b/src/device/tests/qkd/unit/test_application_deployment.py @@ -14,9 +14,11 @@ import pytest import json +import os +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' from device.service.drivers.qkd.QKDDriver2 import QKDDriver -MOCK_QKD_ADDRRESS = '127.0.0.1' +MOCK_QKD_ADDRRESS = '10.0.2.10' MOCK_PORT = 11111 @pytest.fixture diff --git a/src/device/tests/qkd/unit/test_create_apps.py b/src/device/tests/qkd/unit/test_create_apps.py new file mode 100644 index 000000000..c4da21d89 --- /dev/null +++ b/src/device/tests/qkd/unit/test_create_apps.py @@ -0,0 +1,26 @@ +import requests + +QKD_ADDRESS = '10.0.2.10' +QKD_URL = 'http://{:s}/qkd_app/create_qkd_app'.format(QKD_ADDRESS) + +QKD_REQUEST_1 = { + 'app': { + 'server_app_id': '1', + 'client_app_id': [], + 'app_status': 'ON', + 'local_qkdn_id': '00000001-0000-0000-0000-0000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + } +} +print(requests.post(QKD_URL, json=QKD_REQUEST_1)) + +QKD_REQUEST_2 = { + 'app': { + 'server_app_id': '1', + 'client_app_id': [], + 'app_status': 'ON', + 'local_qkdn_id': '00000003-0000-0000-0000-0000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + } +} +print(requests.post(QKD_URL, json=QKD_REQUEST_2)) -- GitLab From 6dce5e7a7a94890ff50767f5928a68d45c3cf067 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:18:18 +0000 Subject: [PATCH 12/15] Fixed mock QKD nodes --- src/tests/tools/mock_qkd_nodes/start.sh | 6 +++--- .../tools/mock_qkd_nodes/{mock.py => wsgi.py} | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) rename src/tests/tools/mock_qkd_nodes/{mock.py => wsgi.py} (97%) diff --git a/src/tests/tools/mock_qkd_nodes/start.sh b/src/tests/tools/mock_qkd_nodes/start.sh index b1bc56d5a..faf2f84ba 100755 --- a/src/tests/tools/mock_qkd_nodes/start.sh +++ b/src/tests/tools/mock_qkd_nodes/start.sh @@ -23,8 +23,8 @@ killbg() { trap killbg EXIT pids=() -flask --app mock run --host 0.0.0.0 --port 11111 & +flask run --host 0.0.0.0 --port 11111 & pids+=($!) -flask --app mock run --host 0.0.0.0 --port 22222 & +flask run --host 0.0.0.0 --port 22222 & pids+=($!) -flask --app mock run --host 0.0.0.0 --port 33333 +flask run --host 0.0.0.0 --port 33333 diff --git a/src/tests/tools/mock_qkd_nodes/mock.py b/src/tests/tools/mock_qkd_nodes/wsgi.py similarity index 97% rename from src/tests/tools/mock_qkd_nodes/mock.py rename to src/tests/tools/mock_qkd_nodes/wsgi.py index 7a606f6ca..3f8847849 100644 --- a/src/tests/tools/mock_qkd_nodes/mock.py +++ b/src/tests/tools/mock_qkd_nodes/wsgi.py @@ -23,7 +23,7 @@ yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) nodes = { - '127.0.0.1:11111': {'node': { + '10.0.2.10:11111': {'node': { 'qkdn_id': '00000001-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -54,7 +54,7 @@ nodes = { { 'qkdi_id': '101', 'qkdi_att_point': { - 'device':'127.0.0.1', + 'device':'10.0.2.10', 'port':'1001' }, 'qkdi_capabilities': { @@ -69,7 +69,7 @@ nodes = { } }, - '127.0.0.1:22222': {'node': { + '10.0.2.10:22222': {'node': { 'qkdn_id': '00000002-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -100,7 +100,7 @@ nodes = { { 'qkdi_id': '201', 'qkdi_att_point': { - 'device':'127.0.0.1', + 'device':'10.0.2.10', 'port':'2001' }, 'qkdi_capabilities': { @@ -109,7 +109,7 @@ nodes = { { 'qkdi_id': '202', 'qkdi_att_point': { - 'device':'127.0.0.1', + 'device':'10.0.2.10', 'port':'2002' }, 'qkdi_capabilities': { @@ -124,7 +124,7 @@ nodes = { } }, - '127.0.0.1:33333': {'node': { + '10.0.2.10:33333': {'node': { 'qkdn_id': '00000003-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -155,7 +155,7 @@ nodes = { { 'qkdi_id': '301', 'qkdi_att_point': { - 'device':'127.0.0.1', + 'device':'10.0.2.10', 'port':'3001' }, 'qkdi_capabilities': { -- GitLab From 9c971c7dd4b2618fd3b8639be25e9958f477281d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:18:31 +0000 Subject: [PATCH 13/15] Updated my_deploy.sh --- my_deploy.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index ee17cb367..5d0d3faaa 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,7 +20,8 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -74,11 +75,11 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # To manage QKD Apps, "service" requires "qkd_app" to be deployed # before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the # "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. -#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then -# BEFORE="${TFS_COMPONENTS% service*}" -# AFTER="${TFS_COMPONENTS#* service}" -# export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" -#fi +if [[ "$TFS_COMPONENTS" == *"service"* ]]; then + BEFORE="${TFS_COMPONENTS% service*}" + AFTER="${TFS_COMPONENTS#* service}" + export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +fi # Set the tag you want to use for your images. -- GitLab From fa52178f630c61e2499e27c377ae5066b6e8e12c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 13:18:48 +0000 Subject: [PATCH 14/15] Fixed WebUI HTTP templates --- src/webui/service/templates/base.html | 4 ++-- src/webui/service/templates/qkd_app/home.html | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/webui/service/templates/base.html b/src/webui/service/templates/base.html index f59e534bd..9cea0a59d 100644 --- a/src/webui/service/templates/base.html +++ b/src/webui/service/templates/base.html @@ -105,9 +105,9 @@
  • {% if '/qkd_app/' in request.path %} - App + QKD Apps {% else %} - App + QKD Apps {% endif %}
  • diff --git a/src/webui/service/templates/qkd_app/home.html b/src/webui/service/templates/qkd_app/home.html index e138acd14..9573013f4 100644 --- a/src/webui/service/templates/qkd_app/home.html +++ b/src/webui/service/templates/qkd_app/home.html @@ -41,7 +41,7 @@ {% for app in apps %} - {{ app.app_id.app_uuid.uuid }} + {{ app.app_id.app_uuid.uuid }} {{ ase.Name(app.app_status).replace('QKDAPPSTATUS_', '') }} @@ -74,12 +74,14 @@ {% endif %} - - - - - - + {% endfor %} -- GitLab From 5d2d4546c3a433f866a992b7323e3d7892ff57cb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 27 Sep 2024 11:20:43 +0000 Subject: [PATCH 15/15] Pre-merge code cleanup --- my_deploy.sh | 13 +++-- src/device/.gitlab-ci.yml | 50 +++++++++---------- src/device/tests/qkd/unit/test_create_apps.py | 14 ++++++ 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index 5d0d3faaa..ee17cb367 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,8 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" -export TFS_COMPONENTS="context device pathcomp service slice nbi webui" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -75,11 +74,11 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # To manage QKD Apps, "service" requires "qkd_app" to be deployed # before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the # "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. -if [[ "$TFS_COMPONENTS" == *"service"* ]]; then - BEFORE="${TFS_COMPONENTS% service*}" - AFTER="${TFS_COMPONENTS#* service}" - export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" -fi +#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then +# BEFORE="${TFS_COMPONENTS% service*}" +# AFTER="${TFS_COMPONENTS#* service}" +# export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +#fi # Set the tag you want to use for your images. diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 402a5a1f8..3ae6b2b20 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -38,29 +38,29 @@ build device: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# Start Mock QKD Nodes before unit testing -start_mock_nodes: - stage: deploy - script: - - bash src/tests/tools/mock_qkd_nodes/start.sh & - - sleep 10 # wait for nodes to spin up - artifacts: - paths: - - mock_nodes.log - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +## Start Mock QKD Nodes before unit testing +#start_mock_nodes: +# stage: deploy +# script: +# - bash src/tests/tools/mock_qkd_nodes/start.sh & +# - sleep 10 # wait for nodes to spin up +# artifacts: +# paths: +# - mock_nodes.log +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -# Prepare Scenario (Start NBI, mock services) -prepare_scenario: - stage: deploy - script: - - pytest src/tests/qkd/unit/PrepareScenario.py - needs: - - start_mock_nodes - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +## Prepare Scenario (Start NBI, mock services) +#prepare_scenario: +# stage: deploy +# script: +# - pytest src/tests/qkd/unit/PrepareScenario.py +# needs: +# - start_mock_nodes +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' # Apply unit test to the component unit_test device: @@ -70,8 +70,8 @@ unit_test device: stage: unit_test needs: - build device - - start_mock_nodes - - prepare_scenario + #- start_mock_nodes + #- prepare_scenario before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - > @@ -94,7 +94,7 @@ unit_test device: - docker logs $IMAGE_NAME - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: diff --git a/src/device/tests/qkd/unit/test_create_apps.py b/src/device/tests/qkd/unit/test_create_apps.py index c4da21d89..4724e5c4a 100644 --- a/src/device/tests/qkd/unit/test_create_apps.py +++ b/src/device/tests/qkd/unit/test_create_apps.py @@ -1,3 +1,17 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import requests QKD_ADDRESS = '10.0.2.10' -- GitLab