From d56ddc3057e823e30fcfdc359c33cfbe562dd6ba Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 17 Oct 2024 12:57:16 +0000 Subject: [PATCH 01/82] Include automatic device tests into pipeline --- src/device/.gitlab-ci.yml | 91 ++++++++++--------- .../tests/qkd/unit/test_mock_qkd_node.py | 10 +- .../tests/qkd/unit/test_qkd_compliance.py | 14 ++- ..._hanling.py => test_qkd_error_handling.py} | 2 +- .../qkd/unit/test_qkd_mock_connectivity.py | 32 +++++-- src/tests/tools/mock_qkd_nodes/wsgi.py | 64 ++----------- 6 files changed, 98 insertions(+), 115 deletions(-) rename src/device/tests/qkd/unit/{test_qkd_error_hanling.py => test_qkd_error_handling.py} (96%) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 3ae6b2b20..16fd27444 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -38,63 +38,65 @@ build device: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -## Start Mock QKD Nodes before unit testing -#start_mock_nodes: -# stage: deploy -# script: -# - bash src/tests/tools/mock_qkd_nodes/start.sh & -# - sleep 10 # wait for nodes to spin up -# artifacts: -# paths: -# - mock_nodes.log -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - -## Prepare Scenario (Start NBI, mock services) -#prepare_scenario: -# stage: deploy -# script: -# - pytest src/tests/qkd/unit/PrepareScenario.py -# needs: -# - start_mock_nodes -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# Deploy mock QKD nodes +deploy_mock_qkd_nodes: + stage: prepare + script: + - echo "Starting mock QKD nodes deployment..." + - > + if docker network list | grep -q teraflowbridge; then + echo "teraflowbridge network is already created"; + else + docker network create -d bridge teraflowbridge; + fi + - > + for port in 11111 22222 33333; do + if lsof -i:$port >/dev/null 2>&1; then + echo "Freeing up port $port..."; + fuser -k $port/tcp; + fi + done + - MOCK_NODES_DIR="/home/cttc-docker/tfs-ctrl/src/tests/tools/mock_qkd_nodes" + - if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" && ./start.sh & + else + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; + fi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/device/**/*.{py,in,yml} + - src/device/Dockerfile + - src/device/tests/*.py + - src/tests/tools/mock_qkd_nodes/** + - .gitlab-ci.yml # Apply unit test to the component -unit_test device: +unit_test_device: variables: IMAGE_NAME: 'device' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: - - build device - #- start_mock_nodes - #- prepare_scenario + - build_device + - deploy_mock_qkd_nodes before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - > - if docker network list | grep teraflowbridge; then - echo "teraflowbridge is already created"; - else - docker network create -d bridge teraflowbridge; - fi - - > - if docker container ls | grep $IMAGE_NAME; then - docker rm -f $IMAGE_NAME; - else - echo "$IMAGE_NAME image is not in the system"; - fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker run --name $IMAGE_NAME --network="host" -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" -e PYTHONPATH="/var/teraflow:/var/teraflow/device:/var/teraflow/tests/tools/mock_qkd_nodes:/var/teraflow/tests" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_mock_connectivity.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_error_handling.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: @@ -110,12 +112,13 @@ unit_test device: - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py - src/$IMAGE_NAME/tests/Dockerfile + - src/tests/tools/mock_qkd_nodes/** - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml ## Deployment of the service in Kubernetes Cluster #deploy device: diff --git a/src/device/tests/qkd/unit/test_mock_qkd_node.py b/src/device/tests/qkd/unit/test_mock_qkd_node.py index f679a8389..afd614fec 100644 --- a/src/device/tests/qkd/unit/test_mock_qkd_node.py +++ b/src/device/tests/qkd/unit/test_mock_qkd_node.py @@ -16,16 +16,20 @@ import pytest import requests from requests.exceptions import ConnectionError +MOCK_QKD_ADDRESS = '127.0.0.1' +MOCK_PORT = 11111 + + def test_mock_qkd_node_responses(): - response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') + response = requests.get(f'http://{MOCK_QKD_ADDRESS}:{MOCK_PORT}/restconf/data/etsi-qkd-sdn-node:qkd_node') assert response.status_code == 200 data = response.json() assert 'qkd_node' in data def test_mock_node_failure_scenarios(): try: - response = requests.get('http://127.0.0.1:12345/restconf/data/etsi-qkd-sdn-node:qkd_node') + response = requests.get(f'http://{MOCK_QKD_ADDRESS}:12345/restconf/data/etsi-qkd-sdn-node:qkd_node') except ConnectionError as e: assert isinstance(e, ConnectionError) else: - pytest.fail("ConnectionError not raised as expected") + pytest.fail("ConnectionError not raised as expected") \ No newline at end of file diff --git a/src/device/tests/qkd/unit/test_qkd_compliance.py b/src/device/tests/qkd/unit/test_qkd_compliance.py index 2f305888e..d4e5cdaf6 100644 --- a/src/device/tests/qkd/unit/test_qkd_compliance.py +++ b/src/device/tests/qkd/unit/test_qkd_compliance.py @@ -15,10 +15,18 @@ import pytest import requests +from requests.exceptions import HTTPError from tests.tools.mock_qkd_nodes.YangValidator import YangValidator def test_compliance_with_yang_models(): validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') - data = response.json() - assert validator.parse_to_dict(data) is not None + try: + response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') + response.raise_for_status() + data = response.json() + assert validator.parse_to_dict(data) is not None, "Data validation failed against YANG model." + except HTTPError as e: + pytest.fail(f"HTTP error occurred: {e}") + except Exception as e: + pytest.fail(f"Unexpected error occurred: {e}") + diff --git a/src/device/tests/qkd/unit/test_qkd_error_hanling.py b/src/device/tests/qkd/unit/test_qkd_error_handling.py similarity index 96% rename from src/device/tests/qkd/unit/test_qkd_error_hanling.py rename to src/device/tests/qkd/unit/test_qkd_error_handling.py index d93e37111..68f4532b3 100644 --- a/src/device/tests/qkd/unit/test_qkd_error_hanling.py +++ b/src/device/tests/qkd/unit/test_qkd_error_handling.py @@ -40,7 +40,7 @@ def test_invalid_operations_on_network_links(qkd_driver): try: # Attempt to perform an invalid operation (simulate wrong resource key) - response = requests.post(f'http://{qkd_driver.address}/invalid_resource', json=invalid_payload) + response = requests.post(f'http://{qkd_driver.address}:{qkd_driver.port}/invalid_resource', json=invalid_payload) response.raise_for_status() except HTTPError as e: diff --git a/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py b/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py index be9427d9b..98a8b6d45 100644 --- a/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py +++ b/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py @@ -12,16 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest, requests +import pytest +import requests +import time +import socket from unittest.mock import patch -from device.service.drivers.qkd.QKDDriver import QKDDriver +from device.service.drivers.qkd.QKDDriver2 import QKDDriver -MOCK_QKD_ADDRRESS = '127.0.0.1' +MOCK_QKD_ADDRESS = '127.0.0.1' # Use localhost to connect to the mock node in the Docker container MOCK_PORT = 11111 +@pytest.fixture(scope="module") +def wait_for_mock_node(): + """ + Fixture to wait for the mock QKD node to be ready before running tests. + """ + timeout = 30 # seconds + start_time = time.time() + while True: + try: + with socket.create_connection((MOCK_QKD_ADDRESS, MOCK_PORT), timeout=1): + break # Success + except (socket.timeout, socket.error): + if time.time() - start_time > timeout: + raise RuntimeError("Timed out waiting for mock QKD node to be ready.") + time.sleep(1) + @pytest.fixture -def qkd_driver(): - return QKDDriver(address=MOCK_QKD_ADDRRESS, port=MOCK_PORT, username='user', password='pass') +def qkd_driver(wait_for_mock_node): + return QKDDriver(address=MOCK_QKD_ADDRESS, port=MOCK_PORT, username='user', password='pass') # Deliverable Test ID: SBI_Test_01 def test_qkd_driver_connection(qkd_driver): @@ -29,7 +48,7 @@ def test_qkd_driver_connection(qkd_driver): # Deliverable Test ID: SBI_Test_01 def test_qkd_driver_invalid_connection(): - qkd_driver = QKDDriver(address='127.0.0.1', port=12345, username='user', password='pass') # Use invalid port directly + qkd_driver = QKDDriver(address=MOCK_QKD_ADDRESS, port=12345, username='user', password='pass') # Use invalid port directly assert qkd_driver.Connect() is False # Deliverable Test ID: SBI_Test_10 @@ -38,4 +57,3 @@ def test_qkd_driver_timeout_connection(mock_get, qkd_driver): mock_get.side_effect = requests.exceptions.Timeout qkd_driver.timeout = 0.001 # Simulate very short timeout assert qkd_driver.Connect() is False - diff --git a/src/tests/tools/mock_qkd_nodes/wsgi.py b/src/tests/tools/mock_qkd_nodes/wsgi.py index 3f8847849..b65cbc420 100644 --- a/src/tests/tools/mock_qkd_nodes/wsgi.py +++ b/src/tests/tools/mock_qkd_nodes/wsgi.py @@ -23,7 +23,7 @@ yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) nodes = { - '10.0.2.10:11111': {'node': { + '127.0.0.1:11111': {'node': { 'qkdn_id': '00000001-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -54,7 +54,7 @@ nodes = { { 'qkdi_id': '101', 'qkdi_att_point': { - 'device':'10.0.2.10', + 'device':'127.0.0.1', 'port':'1001' }, 'qkdi_capabilities': { @@ -69,7 +69,7 @@ nodes = { } }, - '10.0.2.10:22222': {'node': { + '127.0.0.1:22222': {'node': { 'qkdn_id': '00000002-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -100,7 +100,7 @@ nodes = { { 'qkdi_id': '201', 'qkdi_att_point': { - 'device':'10.0.2.10', + 'device':'127.0.0.1', 'port':'2001' }, 'qkdi_capabilities': { @@ -109,7 +109,7 @@ nodes = { { 'qkdi_id': '202', 'qkdi_att_point': { - 'device':'10.0.2.10', + 'device':'127.0.0.1', 'port':'2002' }, 'qkdi_capabilities': { @@ -124,7 +124,7 @@ nodes = { } }, - '10.0.2.10:33333': {'node': { + '127.0.0.1:33333': {'node': { 'qkdn_id': '00000003-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -155,7 +155,7 @@ nodes = { { 'qkdi_id': '301', 'qkdi_att_point': { - 'device':'10.0.2.10', + 'device':'127.0.0.1', 'port':'3001' }, 'qkdi_capabilities': { @@ -316,53 +316,3 @@ def patch(path): reason = str(e) success = False return {'success': success, 'reason': reason} - - - - - -# import json -# from mock import requests -# import pyangbind.lib.pybindJSON as enc -# from pyangbind.lib.serialise import pybindJSONDecoder as dec -# from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node - -# module = etsi_qkd_sdn_node() -# url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:' - -# # Get node all info -# z = requests.get(url).json() -# var = dec.load_json(z, None, None, obj=module) -# print(enc.dumps(var)) - - -# Reset module variable because it is already filled -# module = etsi_qkd_sdn_node() - -# # Get node basic info -# node = module.qkd_node -# z = requests.get(url + 'qkd_node').json() -# var = dec.load_json(z, None, None, obj=node) -# print(enc.dumps(var)) - - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var)) - -# # Edit app 0 -# app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] -# app.client_app_id = 'id_0' -# requests.put(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) - -# # Create app 1 -# app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') -# requests.post(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var)) -- GitLab From 0ed94dd61faa615bc59ae5cdf55541c2e75c856d Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 17 Oct 2024 13:09:42 +0000 Subject: [PATCH 02/82] Fixed bugs --- .gitlab-ci.yml | 1 + src/device/.gitlab-ci.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 873b2ac4b..f24722f71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ stages: #- dependencies - build + - prepare - unit_test - end2end_test diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 16fd27444..001975d88 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,10 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -build device: +build_device: variables: - IMAGE_NAME: 'device' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + IMAGE_NAME: 'device' + IMAGE_TAG: 'latest' stage: build before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY @@ -77,8 +77,8 @@ deploy_mock_qkd_nodes: # Apply unit test to the component unit_test_device: variables: - IMAGE_NAME: 'device' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + IMAGE_NAME: 'device' + IMAGE_TAG: 'latest' stage: unit_test needs: - build_device -- GitLab From 7bbbebe0d3c23790835005dd6f0d2c9676ce51f9 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 17 Oct 2024 13:17:29 +0000 Subject: [PATCH 03/82] Fixed wrong identation in yaml --- src/device/.gitlab-ci.yml | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 001975d88..54915111b 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +stages: + - build + - prepare + - unit_test + # Build, tag, and push the Docker image to the GitLab Docker registry build_device: variables: @@ -30,13 +35,13 @@ build_device: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml # Deploy mock QKD nodes deploy_mock_qkd_nodes: @@ -49,7 +54,7 @@ deploy_mock_qkd_nodes: else docker network create -d bridge teraflowbridge; fi - - > + - | for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..."; @@ -57,7 +62,8 @@ deploy_mock_qkd_nodes: fi done - MOCK_NODES_DIR="/home/cttc-docker/tfs-ctrl/src/tests/tools/mock_qkd_nodes" - - if [ -d "$MOCK_NODES_DIR" ]; then + - | + if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" && ./start.sh & else echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; @@ -66,13 +72,13 @@ deploy_mock_qkd_nodes: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/device/**/*.{py,in,yml} - - src/device/Dockerfile - - src/device/tests/*.py - - src/tests/tools/mock_qkd_nodes/** - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/device/**/*.{py,in,yml} + - src/device/Dockerfile + - src/device/tests/*.py + - src/tests/tools/mock_qkd_nodes/** + - .gitlab-ci.yml # Apply unit test to the component unit_test_device: @@ -106,15 +112,15 @@ unit_test_device: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - src/tests/tools/mock_qkd_nodes/** - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - src/tests/tools/mock_qkd_nodes/** + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml artifacts: when: always reports: -- GitLab From 0034a6b45aacdee1b1a71f637aa6d774a7e3b286 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 17 Oct 2024 13:29:33 +0000 Subject: [PATCH 04/82] Added the correct directory for mocks --- src/device/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 54915111b..56a006cac 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -61,7 +61,7 @@ deploy_mock_qkd_nodes: fuser -k $port/tcp; fi done - - MOCK_NODES_DIR="/home/cttc-docker/tfs-ctrl/src/tests/tools/mock_qkd_nodes" + - MOCK_NODES_DIR="controller/src/tests/tools/mock_qkd_nodes" - | if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" && ./start.sh & -- GitLab From 5b7e5f6b4000ea5aa564fa7fb7ddb3414ae324b5 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 10:45:03 +0000 Subject: [PATCH 05/82] Added service tests --- .gitlab-ci.yml | 48 +++--- src/device/.gitlab-ci.yml | 4 +- src/service/.gitlab-ci.yml | 42 ++++- .../tests/qkd}/descriptorQKD_links.json | 44 +++-- .../tests/qkd/test_functional_bootstrap.py | 152 ++++++++++++++++ src/tests/tools/mock_qkd_nodes/wsgi.py | 163 ++++++++++-------- 6 files changed, 327 insertions(+), 126 deletions(-) rename src/{device/tests/qkd/unit => service/tests/qkd}/descriptorQKD_links.json (68%) create mode 100644 src/service/tests/qkd/test_functional_bootstrap.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f24722f71..e00a85167 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,38 +23,38 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - - local: '/src/monitoring/.gitlab-ci.yml' - - local: '/src/nbi/.gitlab-ci.yml' - - local: '/src/context/.gitlab-ci.yml' + #- local: '/src/monitoring/.gitlab-ci.yml' + #- local: '/src/nbi/.gitlab-ci.yml' + #- local: '/src/context/.gitlab-ci.yml' - local: '/src/device/.gitlab-ci.yml' - local: '/src/service/.gitlab-ci.yml' - - local: '/src/dbscanserving/.gitlab-ci.yml' - - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - - local: '/src/opticalattackdetector/.gitlab-ci.yml' - - local: '/src/opticalattackmanager/.gitlab-ci.yml' - - local: '/src/opticalcontroller/.gitlab-ci.yml' - - local: '/src/ztp/.gitlab-ci.yml' - - local: '/src/policy/.gitlab-ci.yml' - - local: '/src/automation/.gitlab-ci.yml' - - local: '/src/forecaster/.gitlab-ci.yml' + #- local: '/src/dbscanserving/.gitlab-ci.yml' + #- local: '/src/opticalattackmitigator/.gitlab-ci.yml' + #- local: '/src/opticalattackdetector/.gitlab-ci.yml' + #- local: '/src/opticalattackmanager/.gitlab-ci.yml' + #- local: '/src/opticalcontroller/.gitlab-ci.yml' + #- local: '/src/ztp/.gitlab-ci.yml' + #- local: '/src/policy/.gitlab-ci.yml' + #- local: '/src/automation/.gitlab-ci.yml' + #- local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - - local: '/src/slice/.gitlab-ci.yml' + #- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - - local: '/src/pathcomp/.gitlab-ci.yml' + #- local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - - local: '/src/load_generator/.gitlab-ci.yml' - - local: '/src/bgpls_speaker/.gitlab-ci.yml' - - local: '/src/kpi_manager/.gitlab-ci.yml' - - local: '/src/kpi_value_api/.gitlab-ci.yml' - - local: '/src/kpi_value_writer/.gitlab-ci.yml' - - local: '/src/telemetry/.gitlab-ci.yml' - - local: '/src/analytics/.gitlab-ci.yml' - - local: '/src/qos_profile/.gitlab-ci.yml' - - local: '/src/vnt_manager/.gitlab-ci.yml' - - local: '/src/e2e_orchestrator/.gitlab-ci.yml' + #- local: '/src/load_generator/.gitlab-ci.yml' + #- local: '/src/bgpls_speaker/.gitlab-ci.yml' + #- local: '/src/kpi_manager/.gitlab-ci.yml' + #- local: '/src/kpi_value_api/.gitlab-ci.yml' + #- local: '/src/kpi_value_writer/.gitlab-ci.yml' + #- local: '/src/telemetry/.gitlab-ci.yml' + #- local: '/src/analytics/.gitlab-ci.yml' + #- local: '/src/qos_profile/.gitlab-ci.yml' + #- local: '/src/vnt_manager/.gitlab-ci.yml' + #- local: '/src/e2e_orchestrator/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 56a006cac..5337a5800 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -97,8 +97,8 @@ unit_test_device: - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_mock_connectivity.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 8eda9d2be..35897b5da 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -86,7 +86,7 @@ unit_test service: docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - docker logs nats @@ -158,6 +158,46 @@ unit_test service: "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + # QKD Mock Nodes Deployment + - echo "Starting stage: deploy_mock_nodes" + - for port in 11111 22222 33333; do + if lsof -i:$port >/dev/null 2>&1; then + echo "Freeing up port $port..." + fuser -k $port/tcp + fi + done + - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" + - if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" || exit + ./start.sh & + MOCK_NODES_PID=$! + else + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." + exit 1 + fi + - echo "Waiting for mock nodes to be up..." + - RETRY_COUNT=0 + - MAX_RETRIES=15 + - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if curl -s http://127.0.0.1:11111 > /dev/null && \ + curl -s http://127.0.0.1:22222 > /dev/null && \ + curl -s http://127.0.0.1:33333 > /dev/null; then + echo "Mock nodes are up!" + break + else + echo "Mock nodes not ready, retrying in 5 seconds..." + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 5 + fi + done + - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Error: Mock nodes failed to start after multiple attempts." + exit 1 + fi + + # Run QKD Bootstrap Test + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: # Check status after the tests diff --git a/src/device/tests/qkd/unit/descriptorQKD_links.json b/src/service/tests/qkd/descriptorQKD_links.json similarity index 68% rename from src/device/tests/qkd/unit/descriptorQKD_links.json rename to src/service/tests/qkd/descriptorQKD_links.json index 28a9e7d5a..d80864cb0 100644 --- a/src/device/tests/qkd/unit/descriptorQKD_links.json +++ b/src/service/tests/qkd/descriptorQKD_links.json @@ -10,68 +10,64 @@ "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - }, { "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - }, - { + { "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - } ], "links": [ - { - "link_id": {"link_uuid": {"uuid": "QKD1/10.0.2.10:1001==QKD2/10.0.2.10:2001"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD1/:1001==QKD2/:2001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}} + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": ":1001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2001"}} ] }, { - "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2001==QKD1/10.0.2.10:1001"}}, + "link_id": {"link_uuid": {"uuid": "QKD2/:2001==QKD1/:1001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": ":1001"}} ] }, - { - "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2002==QKD3/10.0.2.10:3001"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD2/:2002==QKD3/:3001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}}, - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2002"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": ":3001"}} ] }, - { - "link_id": {"link_uuid": {"uuid": "QKD3/10.0.2.10:3001==QKD2/10.0.2.10:2002"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD3/:3001==QKD2/:2002"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}} + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": ":3001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2002"}} ] } - ] -} +} \ No newline at end of file diff --git a/src/service/tests/qkd/test_functional_bootstrap.py b/src/service/tests/qkd/test_functional_bootstrap.py new file mode 100644 index 000000000..daf35f0de --- /dev/null +++ b/src/service/tests/qkd/test_functional_bootstrap.py @@ -0,0 +1,152 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time, json, socket, re +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +# Update the path to your QKD descriptor file +DESCRIPTOR_FILE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'descriptorQKD_links.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def load_descriptor_with_runtime_ip(descriptor_file_path): + """ + Load the descriptor file and replace placeholder IP with the machine's IP address. + """ + with open(descriptor_file_path, 'r') as descriptor_file: + descriptor = descriptor_file.read() + + # Get the current machine's IP address + try: + # Use socket to get the local IP address directly from the network interface + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + current_ip = s.getsockname()[0] + s.close() + except Exception as e: + raise Exception(f"Unable to get the IP address: {str(e)}") + + # Replace all occurrences of with the current IP + updated_descriptor = re.sub(r"", current_ip, descriptor) + + # Write updated descriptor back + with open(descriptor_file_path, 'w') as descriptor_file: + descriptor_file.write(updated_descriptor) + + return json.loads(updated_descriptor) + +def load_and_process_descriptor(context_client, device_client, descriptor_file_path): + """ + Function to load and process descriptor programmatically, similar to what WebUI does. + """ + print(f"Loading descriptor from file: {descriptor_file_path}") + try: + # Update the descriptor with the runtime IP address + descriptor = load_descriptor_with_runtime_ip(descriptor_file_path) + + # Initialize DescriptorLoader with the updated descriptor file + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file_path, context_client=context_client, device_client=device_client + ) + + # Process and validate the descriptor + print("Processing the descriptor...") + results = descriptor_loader.process() + print(f"Descriptor processing results: {results}") + + print("Checking descriptor load results...") + check_descriptor_load_results(results, descriptor_loader) + + print("Validating descriptor...") + descriptor_loader.validate() + print("Descriptor validated successfully.") + except Exception as e: + LOGGER.error(f"Failed to load and process descriptor: {e}") + raise e + +def test_qkd_scenario_bootstrap( + context_client: ContextClient, # pylint: disable=redefined-outer-name + device_client: DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the QKD scenario is correctly bootstrapped. + """ + print("Starting QKD scenario bootstrap test...") + + # Check if context_client and device_client are instantiated + if context_client is None: + print("Error: context_client is not instantiated!") + else: + print(f"context_client is instantiated: {context_client}") + + if device_client is None: + print("Error: device_client is not instantiated!") + else: + print(f"device_client is instantiated: {device_client}") + + # Validate empty scenario + print("Validating empty scenario...") + validate_empty_scenario(context_client) + + # Load the descriptor + load_and_process_descriptor(context_client, device_client, DESCRIPTOR_FILE_PATH) + +def test_qkd_devices_enabled( + context_client: ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the QKD devices are enabled. + """ + print("Starting QKD devices enabled test...") + + # Check if context_client is instantiated + if context_client is None: + print("Error: context_client is not instantiated!") + else: + print(f"context_client is instantiated: {context_client}") + + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + + while (num_devices != num_devices_enabled) and (num_retry < 10): + print(f"Attempt {num_retry + 1}: Checking device status...") + + time.sleep(1.0) # Add a delay to allow for device enablement + + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + print(f"Total devices found: {num_devices}") + + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status == DEVICE_OP_STATUS_ENABLED: + num_devices_enabled += 1 + + print(f"Devices enabled: {num_devices_enabled}/{num_devices}") + num_retry += 1 + + # Final check to ensure all devices are enabled + print(f"Final device status: {num_devices_enabled}/{num_devices} devices enabled.") + assert num_devices_enabled == num_devices + print("QKD devices enabled test completed.") \ No newline at end of file diff --git a/src/tests/tools/mock_qkd_nodes/wsgi.py b/src/tests/tools/mock_qkd_nodes/wsgi.py index b65cbc420..2a3d48adb 100644 --- a/src/tests/tools/mock_qkd_nodes/wsgi.py +++ b/src/tests/tools/mock_qkd_nodes/wsgi.py @@ -13,17 +13,20 @@ # limitations under the License. import os +import socket from flask import Flask, request from YangValidator import YangValidator app = Flask(__name__) +# Retrieve the IP address of the current machine +current_ip = socket.gethostbyname(socket.gethostname()) yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - +# Update IP address with the current machine's IP nodes = { - '127.0.0.1:11111': {'node': { + f'{current_ip}:11111': {'node': { 'qkdn_id': '00000001-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -31,7 +34,7 @@ nodes = { 'qkd_applications': { 'qkd_app': [ { - 'app_id': '00000001-0001-0000-0000-000000000000', + 'app_id': '00000001-0001-0000-0000-000000000000', 'client_app_id': [], 'app_statistics': { 'statistics': [] @@ -54,8 +57,8 @@ nodes = { { 'qkdi_id': '101', 'qkdi_att_point': { - 'device':'127.0.0.1', - 'port':'1001' + 'device': current_ip, + 'port': '1001' }, 'qkdi_capabilities': { } @@ -63,13 +66,11 @@ nodes = { ] }, 'qkd_links': { - 'qkd_link': [ - - ] + 'qkd_link': [] } }, - '127.0.0.1:22222': {'node': { + f'{current_ip}:22222': {'node': { 'qkdn_id': '00000002-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -77,7 +78,7 @@ nodes = { 'qkd_applications': { 'qkd_app': [ { - 'app_id': '00000002-0001-0000-0000-000000000000', + 'app_id': '00000002-0001-0000-0000-000000000000', 'client_app_id': [], 'app_statistics': { 'statistics': [] @@ -100,8 +101,8 @@ nodes = { { 'qkdi_id': '201', 'qkdi_att_point': { - 'device':'127.0.0.1', - 'port':'2001' + 'device': current_ip, + 'port': '2001' }, 'qkdi_capabilities': { } @@ -109,8 +110,8 @@ nodes = { { 'qkdi_id': '202', 'qkdi_att_point': { - 'device':'127.0.0.1', - 'port':'2002' + 'device': current_ip, + 'port': '2002' }, 'qkdi_capabilities': { } @@ -118,13 +119,11 @@ nodes = { ] }, 'qkd_links': { - 'qkd_link': [ - - ] + 'qkd_link': [] } }, - '127.0.0.1:33333': {'node': { + f'{current_ip}:33333': {'node': { 'qkdn_id': '00000003-0000-0000-0000-000000000000', }, 'qkdn_capabilities': { @@ -132,7 +131,7 @@ nodes = { 'qkd_applications': { 'qkd_app': [ { - 'app_id': '00000003-0001-0000-0000-000000000000', + 'app_id': '00000003-0001-0000-0000-000000000000', 'client_app_id': [], 'app_statistics': { 'statistics': [] @@ -155,8 +154,8 @@ nodes = { { 'qkdi_id': '301', 'qkdi_att_point': { - 'device':'127.0.0.1', - 'port':'3001' + 'device': current_ip, + 'port': '3001' }, 'qkdi_capabilities': { } @@ -164,18 +163,20 @@ nodes = { ] }, 'qkd_links': { - 'qkd_link': [ - - ] + 'qkd_link': [] } } } - def get_side_effect(url): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - ip_port, _, _, header, *steps = steps.split('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") + + ip_port, _, _, header, *steps = parts header_splitted = header.split(':') @@ -197,35 +198,18 @@ def get_side_effect(url): if not steps: return tree, tree - endpoint, *steps = steps value = nodes[ip_port][endpoint] if not steps: - return_value = {endpoint:value} + return_value = {endpoint: value} tree['qkd_node'].update(return_value) return return_value, tree - - - ''' - element, *steps = steps - - container, key = element.split('=') - - # value = value[container][key] - - if not steps: - return_value['qkd_node'][endpoint] = [value] - return return_value - - ''' raise Exception('Url too long') - - def edit(from_dict, to_dict, create): for key, value in from_dict.items(): if isinstance(value, dict): @@ -237,11 +221,15 @@ def edit(from_dict, to_dict, create): else: to_dict[key] = value - - def edit_side_effect(url, json, create): steps = url.lstrip('https://').lstrip('http://').rstrip('/') - ip_port, _, _, header, *steps = steps.split('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") + + ip_port, _, _, header, *steps = parts module, root = header.split(':') @@ -249,7 +237,7 @@ def edit_side_effect(url, json, create): assert(root == 'qkd_node') if not steps: - edit(json, nodes[ip_port]['node']) + edit(json, nodes[ip_port]['node'], create) return endpoint, *steps = steps @@ -258,36 +246,19 @@ def edit_side_effect(url, json, create): edit(json[endpoint], nodes[ip_port][endpoint], create) return - - ''' - element, *steps = steps - - container, key = element.split('=') - - if not steps: - if key not in nodes[ip_port][endpoint][container] and create: - nodes[ip_port][endpoint][container][key] = {} - - edit(json, nodes[ip_port][endpoint][container][key], create) - return 0 - ''' - raise Exception('Url too long') - - - - - @app.get('/', defaults={'path': ''}) @app.get("/") @app.get('/') def get(path): - msg, msg_validate = get_side_effect(request.base_url) - print(msg_validate) - yang_validator.parse_to_dict(msg_validate) - return msg - + try: + msg, msg_validate = get_side_effect(request.base_url) + print(msg_validate) + yang_validator.parse_to_dict(msg_validate) + return msg + except ValueError as e: + return {'error': str(e)}, 400 @app.post('/', defaults={'path': ''}) @app.post("/") @@ -301,8 +272,6 @@ def post(path): reason = str(e) success = False return {'success': success, 'reason': reason} - - @app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) @app.route("/", methods=['PUT', 'PATCH']) @@ -316,3 +285,47 @@ def patch(path): reason = str(e) success = False return {'success': success, 'reason': reason} + +# import json +# from mock import requests +# import pyangbind.lib.pybindJSON as enc +# from pyangbind.lib.serialise import pybindJSONDecoder as dec +# from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node + +# module = etsi_qkd_sdn_node() +# url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:' + +# # Get node all info +# z = requests.get(url).json() +# var = dec.load_json(z, None, None, obj=module) +# print(enc.dumps(var)) + +# Reset module variable because it is already filled +# module = etsi_qkd_sdn_node() + +# # Get node basic info +# node = module.qkd_node +# z = requests.get(url + 'qkd_node').json() +# var = dec.load_json(z, None, None, obj=node) +# print(enc.dumps(var)) + +# # Get all apps +# apps = node.qkd_applications +# z = requests.get(url + 'qkd_node/qkd_applications').json() +# var = dec.load_json(z, None, None, obj=apps) +# print(enc.dumps(var)) + +# # Edit app 0 +# app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] +# app.client_app_id = 'id_0' +# requests.put(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) + +# # Create app 1 +# app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') +# requests.post(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) + +# # Get all apps +# apps = node.qkd_applications +# z = requests.get(url + 'qkd_node/qkd_applications').json() +# var = dec.load_json(z, None, None, obj=apps) +# print(enc.dumps(var)) -- GitLab From e828c9ce63d90441c5d572d5943ac21e045eb6c8 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 10:50:35 +0000 Subject: [PATCH 06/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 35897b5da..bae51e83d 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -167,7 +167,8 @@ unit_test service: fi done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - if [ -d "$MOCK_NODES_DIR" ]; then + - | + if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit ./start.sh & MOCK_NODES_PID=$! @@ -178,7 +179,8 @@ unit_test service: - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + - | + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ curl -s http://127.0.0.1:33333 > /dev/null; then @@ -190,7 +192,8 @@ unit_test service: sleep 5 fi done - - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + - | + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Error: Mock nodes failed to start after multiple attempts." exit 1 fi -- GitLab From e7e3fec4a57bdc230310cf2e81af6fa9967f8b9b Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 10:53:25 +0000 Subject: [PATCH 07/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 151 ++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 84 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index bae51e83d..83d01a903 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,23 +49,22 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - # Context-related - - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - - # Device-related - - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - - # Pathcomp-related - - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi - - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - - # Service-related - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - + - | + # Context-related + if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + - | + # Device-related + if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi + - | + # Pathcomp-related + if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi + if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi + - | + # Service-related + if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" @@ -74,17 +73,16 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - # Context preparation - - docker volume create crdb - - > - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 - --volume "crdb:/cockroach/cockroach-data" - cockroachdb/cockroach:latest-v22.2 start-single-node - - > - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 - nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - | + # Context preparation + docker volume create crdb + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ + --volume "crdb:/cockroach/cockroach-data" \ + cockroachdb/cockroach:latest-v22.2 start-single-node + - | + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ + nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - docker logs crdb @@ -95,55 +93,50 @@ unit_test service: - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS - - > - docker run --name context -d -p 1010:1010 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - | + docker run --name context -d -p 1010:1010 \ + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ + --env "MB_BACKEND=nats" \ + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" \ + --network=teraflowbridge \ + $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CONTEXTSERVICE_SERVICE_HOST - - # Device preparation - - > - docker run --name device -d -p 2020:2020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/device:$IMAGE_TAG + - | + # Device preparation + docker run --name device -d -p 2020:2020 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --network=teraflowbridge \ + $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $DEVICESERVICE_SERVICE_HOST - - # PathComp preparation - - > - docker run --name pathcomp-backend -d -p 8081:8081 - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG + - | + # PathComp preparation + docker run --name pathcomp-backend -d -p 8081:8081 \ + --network=teraflowbridge \ + $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMP_BACKEND_HOST - sleep 1 - - > - docker run --name pathcomp-frontend -d -p 10020:10020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" - --env "PATHCOMP_BACKEND_PORT=8081" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG + - | + docker run --name pathcomp-frontend -d -p 10020:10020 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ + --env "PATHCOMP_BACKEND_PORT=8081" \ + --network=teraflowbridge \ + $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - sleep 1 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMPSERVICE_SERVICE_HOST - - # Service preparation - - > - docker run --name $IMAGE_NAME -d -p 3030:3030 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - # Check status before the tests + - | + # Service preparation + docker run --name $IMAGE_NAME -d -p 3030:3030 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" \ + --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" \ + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" \ + --network=teraflowbridge \ + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs context @@ -151,16 +144,14 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - # Run the tests - - > - docker exec -i $IMAGE_NAME bash -c - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - | + # Run the tests + docker exec -i $IMAGE_NAME bash -c \ + "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - - # QKD Mock Nodes Deployment - echo "Starting stage: deploy_mock_nodes" - - for port in 11111 22222 33333; do + - | + for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..." fuser -k $port/tcp @@ -197,32 +188,25 @@ unit_test service: echo "Error: Mock nodes failed to start after multiple attempts." exit 1 fi - - # Run QKD Bootstrap Test - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - # Check status after the tests - docker ps -a - docker logs context - docker logs device - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - docker rm -f $IMAGE_NAME - docker rm -f pathcomp-frontend - docker rm -f pathcomp-backend - docker rm -f device - docker rm -f context - - docker rm -f $IMAGE_NAME crdb nats - docker volume rm -f crdb - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force - rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' @@ -234,7 +218,6 @@ unit_test service: - src/$IMAGE_NAME/tests/*.py - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml - artifacts: when: always reports: @@ -264,4 +247,4 @@ unit_test service: # - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' # when: manual # - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -# when: manual +# when: manual \ No newline at end of file -- GitLab From 50476c7554e9ce5a2ccf05c4fa4493d877566e2f Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 10:59:04 +0000 Subject: [PATCH 08/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 115 +++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 83d01a903..7209fed22 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -48,22 +48,27 @@ unit_test service: - build service before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - | - # Context-related - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - > + if docker network list | grep teraflowbridge; then + echo "teraflowbridge is already created"; + else + docker network create --driver=bridge teraflowbridge; + fi + - > + # Context-related cleanup + if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi; + if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi; if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - - | - # Device-related - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + - > + # Device-related cleanup + if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi; if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - - | - # Pathcomp-related - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi + - > + # Pathcomp-related cleanup + if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi; if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - - | - # Service-related + - > + # Service-related cleanup if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" @@ -73,27 +78,24 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - | + - > # Context preparation - docker volume create crdb + docker volume create crdb; docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ --volume "crdb:/cockroach/cockroach-data" \ cockroachdb/cockroach:latest-v22.2 start-single-node - - | + - > docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - docker logs nats - - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $NATS_ADDRESS - - | + - echo "CRDB Address: $CRDB_ADDRESS" + - echo "NATS Address: $NATS_ADDRESS" + - > docker run --name context -d -p 1010:1010 \ --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ --env "MB_BACKEND=nats" \ @@ -101,34 +103,33 @@ unit_test service: --network=teraflowbridge \ $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CONTEXTSERVICE_SERVICE_HOST - - | + - echo "Context Service Host: $CONTEXTSERVICE_SERVICE_HOST" + - > # Device preparation docker run --name device -d -p 2020:2020 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ --network=teraflowbridge \ $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $DEVICESERVICE_SERVICE_HOST - - | + - echo "Device Service Host: $DEVICESERVICE_SERVICE_HOST" + - > # PathComp preparation docker run --name pathcomp-backend -d -p 8081:8081 \ --network=teraflowbridge \ $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMP_BACKEND_HOST - - sleep 1 - - | + - echo "PathComp Backend Host: $PATHCOMP_BACKEND_HOST" + - > docker run --name pathcomp-frontend -d -p 10020:10020 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ --env "PATHCOMP_BACKEND_PORT=8081" \ --network=teraflowbridge \ $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - - sleep 1 + - sleep 5 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMPSERVICE_SERVICE_HOST - - | + - echo "PathComp Frontend Host: $PATHCOMPSERVICE_SERVICE_HOST" + - > # Service preparation docker run --name $IMAGE_NAME -d -p 3030:3030 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ @@ -144,49 +145,49 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - | + - > # Run the tests docker exec -i $IMAGE_NAME bash -c \ "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - echo "Starting stage: deploy_mock_nodes" - - | + - > for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..." - fuser -k $port/tcp - fi + echo "Freeing up port $port..."; + fuser -k $port/tcp; + fi; done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - | + - > if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit - ./start.sh & - MOCK_NODES_PID=$! + cd "$MOCK_NODES_DIR" || exit; + ./start.sh &; + MOCK_NODES_PID=$!; else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." - exit 1 + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; + exit 1; fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - | + - > while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!" - break + echo "Mock nodes are up!"; + break; else - echo "Mock nodes not ready, retrying in 5 seconds..." - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 5 - fi + echo "Mock nodes not ready, retrying in 5 seconds..."; + RETRY_COUNT=$((RETRY_COUNT + 1)); + sleep 5; + fi; done - - | + - > if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts." - exit 1 + echo "Error: Mock nodes failed to start after multiple attempts."; + exit 1; fi - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' @@ -219,9 +220,9 @@ unit_test service: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy service: @@ -247,4 +248,4 @@ unit_test service: # - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' # when: manual # - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -# when: manual \ No newline at end of file +# when: manual -- GitLab From 84c8e9851cc5a0ab916d5962641abd1dd7ef0c11 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 11:01:57 +0000 Subject: [PATCH 09/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 88 +++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 7209fed22..bffd94385 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -48,28 +48,28 @@ unit_test service: - build service before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - > + - | if docker network list | grep teraflowbridge; then - echo "teraflowbridge is already created"; + echo "teraflowbridge is already created" else - docker network create --driver=bridge teraflowbridge; + docker network create --driver=bridge teraflowbridge fi - - > + - | # Context-related cleanup - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi; - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi; - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - - > + if docker container ls | grep crdb; then docker rm -f crdb; fi + if docker volume ls | grep crdb; then docker volume rm -f crdb; fi + if docker container ls | grep nats; then docker rm -f nats; fi + - | # Device-related cleanup - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi; - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - - > + if docker container ls | grep context; then docker rm -f context; fi + if docker container ls | grep device; then docker rm -f device; fi + - | # Pathcomp-related cleanup - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi; - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - - > + if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; fi + if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; fi + - | # Service-related cleanup - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" @@ -78,14 +78,14 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - > + - | # Context preparation - docker volume create crdb; + docker volume create crdb docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ --volume "crdb:/cockroach/cockroach-data" \ cockroachdb/cockroach:latest-v22.2 start-single-node - - > + - | docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." @@ -95,7 +95,7 @@ unit_test service: - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo "CRDB Address: $CRDB_ADDRESS" - echo "NATS Address: $NATS_ADDRESS" - - > + - | docker run --name context -d -p 1010:1010 \ --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ --env "MB_BACKEND=nats" \ @@ -104,7 +104,7 @@ unit_test service: $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo "Context Service Host: $CONTEXTSERVICE_SERVICE_HOST" - - > + - | # Device preparation docker run --name device -d -p 2020:2020 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ @@ -112,14 +112,14 @@ unit_test service: $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo "Device Service Host: $DEVICESERVICE_SERVICE_HOST" - - > + - | # PathComp preparation docker run --name pathcomp-backend -d -p 8081:8081 \ --network=teraflowbridge \ $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo "PathComp Backend Host: $PATHCOMP_BACKEND_HOST" - - > + - | docker run --name pathcomp-frontend -d -p 10020:10020 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ @@ -129,7 +129,7 @@ unit_test service: - sleep 5 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo "PathComp Frontend Host: $PATHCOMPSERVICE_SERVICE_HOST" - - > + - | # Service preparation docker run --name $IMAGE_NAME -d -p 3030:3030 \ --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ @@ -145,49 +145,49 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - > + - | # Run the tests docker exec -i $IMAGE_NAME bash -c \ "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - echo "Starting stage: deploy_mock_nodes" - - > + - | for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..."; - fuser -k $port/tcp; - fi; + echo "Freeing up port $port..." + fuser -k $port/tcp + fi done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - > + - | if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit; - ./start.sh &; - MOCK_NODES_PID=$!; + cd "$MOCK_NODES_DIR" || exit + ./start.sh & + MOCK_NODES_PID=$! else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; - exit 1; + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." + exit 1 fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - > + - | while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!"; - break; + echo "Mock nodes are up!" + break else - echo "Mock nodes not ready, retrying in 5 seconds..."; - RETRY_COUNT=$((RETRY_COUNT + 1)); - sleep 5; - fi; + echo "Mock nodes not ready, retrying in 5 seconds..." + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 5 + fi done - - > + - | if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts."; - exit 1; + echo "Error: Mock nodes failed to start after multiple attempts." + exit 1 fi - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' -- GitLab From 12697f4c6e5d9c9dc12342d46eb52ef3ee9ebec0 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 11:07:32 +0000 Subject: [PATCH 10/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 222 +++++++++++++++++++++---------------- 1 file changed, 128 insertions(+), 94 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index bffd94385..41a7ffad2 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -48,28 +48,64 @@ unit_test service: - build service before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - | + - > if docker network list | grep teraflowbridge; then - echo "teraflowbridge is already created" + echo "teraflowbridge is already created"; else - docker network create --driver=bridge teraflowbridge + docker network create --driver=bridge teraflowbridge; + fi + - > + # Context-related + if docker container ls | grep crdb; then + docker rm -f crdb; + else + echo "CockroachDB container is not in the system"; + fi + - > + if docker volume ls | grep crdb; then + docker volume rm -f crdb; + else + echo "CockroachDB volume is not in the system"; + fi + - > + if docker container ls | grep nats; then + docker rm -f nats; + else + echo "NATS container is not in the system"; + fi + - > + # Device-related + if docker container ls | grep context; then + docker rm -f context; + else + echo "context image is not in the system"; + fi + - > + if docker container ls | grep device; then + docker rm -f device; + else + echo "device image is not in the system"; + fi + - > + # Pathcomp-related + if docker container ls | grep pathcomp-frontend; then + docker rm -f pathcomp-frontend; + else + echo "pathcomp-frontend image is not in the system"; + fi + - > + if docker container ls | grep pathcomp-backend; then + docker rm -f pathcomp-backend; + else + echo "pathcomp-backend image is not in the system"; + fi + - > + # Service-related + if docker container ls | grep $IMAGE_NAME; then + docker rm -f $IMAGE_NAME; + else + echo "$IMAGE_NAME image is not in the system"; fi - - | - # Context-related cleanup - if docker container ls | grep crdb; then docker rm -f crdb; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; fi - if docker container ls | grep nats; then docker rm -f nats; fi - - | - # Device-related cleanup - if docker container ls | grep context; then docker rm -f context; fi - if docker container ls | grep device; then docker rm -f device; fi - - | - # Pathcomp-related cleanup - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; fi - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; fi - - | - # Service-related cleanup - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" @@ -78,66 +114,66 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - | - # Context preparation - docker volume create crdb - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ - --volume "crdb:/cockroach/cockroach-data" \ - cockroachdb/cockroach:latest-v22.2 start-single-node - - | - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ - nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - > + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 + nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + - docker logs nats + - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo "CRDB Address: $CRDB_ADDRESS" - - echo "NATS Address: $NATS_ADDRESS" - - | - docker run --name context -d -p 1010:1010 \ - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ - --env "MB_BACKEND=nats" \ - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" \ - --network=teraflowbridge \ - $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - echo $NATS_ADDRESS + - > + docker run --name context -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "MB_BACKEND=nats" + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo "Context Service Host: $CONTEXTSERVICE_SERVICE_HOST" - - | - # Device preparation - docker run --name device -d -p 2020:2020 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --network=teraflowbridge \ - $CI_REGISTRY_IMAGE/device:$IMAGE_TAG + - echo $CONTEXTSERVICE_SERVICE_HOST + - > + docker run --name device -d -p 2020:2020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo "Device Service Host: $DEVICESERVICE_SERVICE_HOST" - - | - # PathComp preparation - docker run --name pathcomp-backend -d -p 8081:8081 \ - --network=teraflowbridge \ - $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG + - echo $DEVICESERVICE_SERVICE_HOST + - > + docker run --name pathcomp-backend -d -p 8081:8081 + --network=teraflowbridge + $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo "PathComp Backend Host: $PATHCOMP_BACKEND_HOST" - - | - docker run --name pathcomp-frontend -d -p 10020:10020 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ - --env "PATHCOMP_BACKEND_PORT=8081" \ - --network=teraflowbridge \ - $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - - sleep 5 + - echo $PATHCOMP_BACKEND_HOST + - sleep 1 + - > + docker run --name pathcomp-frontend -d -p 10020:10020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" + --env "PATHCOMP_BACKEND_PORT=8081" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG + - sleep 1 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo "PathComp Frontend Host: $PATHCOMPSERVICE_SERVICE_HOST" - - | - # Service preparation - docker run --name $IMAGE_NAME -d -p 3030:3030 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" \ - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" \ - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" \ - --network=teraflowbridge \ - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - echo $PATHCOMPSERVICE_SERVICE_HOST + - > + docker run --name $IMAGE_NAME -d -p 3030:3030 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" + --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs context @@ -145,49 +181,47 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - | - # Run the tests - docker exec -i $IMAGE_NAME bash -c \ - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - echo "Starting stage: deploy_mock_nodes" - - | - for port in 11111 22222 33333; do + - for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..." - fuser -k $port/tcp + echo "Freeing up port $port..."; + fuser -k $port/tcp; fi done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - | + - > if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit - ./start.sh & - MOCK_NODES_PID=$! + cd "$MOCK_NODES_DIR" || exit; + ./start.sh &; + MOCK_NODES_PID=$!; else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." - exit 1 + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; + exit 1; fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - | + - > while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!" - break + echo "Mock nodes are up!"; + break; else - echo "Mock nodes not ready, retrying in 5 seconds..." - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 5 + echo "Mock nodes not ready, retrying in 5 seconds..."; + RETRY_COUNT=$((RETRY_COUNT + 1)); + sleep 5; fi done - - | + - > if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts." - exit 1 + echo "Error: Mock nodes failed to start after multiple attempts."; + exit 1; fi - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' @@ -203,7 +237,7 @@ unit_test service: - docker rm -f pathcomp-backend - docker rm -f device - docker rm -f context - - docker rm -f $IMAGE_NAME crdb nats + - docker rm -f crdb nats - docker volume rm -f crdb - docker network rm teraflowbridge - docker volume prune --force -- GitLab From 095d634d6d0c6a70ead76c2ff4fb2a4c1fdcf9b1 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 11:12:32 +0000 Subject: [PATCH 11/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 155 +++++-------------------------------- 1 file changed, 21 insertions(+), 134 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 41a7ffad2..4c76afe71 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -48,64 +48,15 @@ unit_test service: - build service before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - > - if docker network list | grep teraflowbridge; then - echo "teraflowbridge is already created"; - else - docker network create --driver=bridge teraflowbridge; - fi - - > - # Context-related - if docker container ls | grep crdb; then - docker rm -f crdb; - else - echo "CockroachDB container is not in the system"; - fi - - > - if docker volume ls | grep crdb; then - docker volume rm -f crdb; - else - echo "CockroachDB volume is not in the system"; - fi - - > - if docker container ls | grep nats; then - docker rm -f nats; - else - echo "NATS container is not in the system"; - fi - - > - # Device-related - if docker container ls | grep context; then - docker rm -f context; - else - echo "context image is not in the system"; - fi - - > - if docker container ls | grep device; then - docker rm -f device; - else - echo "device image is not in the system"; - fi - - > - # Pathcomp-related - if docker container ls | grep pathcomp-frontend; then - docker rm -f pathcomp-frontend; - else - echo "pathcomp-frontend image is not in the system"; - fi - - > - if docker container ls | grep pathcomp-backend; then - docker rm -f pathcomp-backend; - else - echo "pathcomp-backend image is not in the system"; - fi - - > - # Service-related - if docker container ls | grep $IMAGE_NAME; then - docker rm -f $IMAGE_NAME; - else - echo "$IMAGE_NAME image is not in the system"; - fi + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi + - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi + - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" @@ -115,14 +66,8 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - docker volume create crdb - - > - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 - --volume "crdb:/cockroach/cockroach-data" - cockroachdb/cockroach:latest-v22.2 start-single-node - - > - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 - nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 --volume "crdb:/cockroach/cockroach-data" cockroachdb/cockroach:latest-v22.2 start-single-node + - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - docker logs crdb @@ -133,47 +78,21 @@ unit_test service: - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS - - > - docker run --name context -d -p 1010:1010 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - docker run --name context -d -p 1010:1010 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --env "MB_BACKEND=nats" --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --network=teraflowbridge $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CONTEXTSERVICE_SERVICE_HOST - - > - docker run --name device -d -p 2020:2020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/device:$IMAGE_TAG + - docker run --name device -d -p 2020:2020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --network=teraflowbridge $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $DEVICESERVICE_SERVICE_HOST - - > - docker run --name pathcomp-backend -d -p 8081:8081 - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG + - docker run --name pathcomp-backend -d -p 8081:8081 --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMP_BACKEND_HOST - sleep 1 - - > - docker run --name pathcomp-frontend -d -p 10020:10020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" - --env "PATHCOMP_BACKEND_PORT=8081" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG + - docker run --name pathcomp-frontend -d -p 10020:10020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" --env "PATHCOMP_BACKEND_PORT=8081" --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - sleep 1 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMPSERVICE_SERVICE_HOST - - > - docker run --name $IMAGE_NAME -d -p 3030:3030 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker run --name $IMAGE_NAME -d -p 3030:3030 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs context @@ -181,48 +100,17 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - > - docker exec -i $IMAGE_NAME bash -c - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - echo "Starting stage: deploy_mock_nodes" - - for port in 11111 22222 33333; do - if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..."; - fuser -k $port/tcp; - fi - done + - for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..."; fuser -k $port/tcp; fi; done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - > - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit; - ./start.sh &; - MOCK_NODES_PID=$!; - else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; - exit 1; - fi + - if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit; ./start.sh &; MOCK_NODES_PID=$!; else echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; exit 1; fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - > - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - if curl -s http://127.0.0.1:11111 > /dev/null && \ - curl -s http://127.0.0.1:22222 > /dev/null && \ - curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!"; - break; - else - echo "Mock nodes not ready, retrying in 5 seconds..."; - RETRY_COUNT=$((RETRY_COUNT + 1)); - sleep 5; - fi - done - - > - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts."; - exit 1; - fi + - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && curl -s http://127.0.0.1:22222 > /dev/null && curl -s http://127.0.0.1:33333 > /dev/null; then echo "Mock nodes are up!"; break; else echo "Mock nodes not ready, retrying in 5 seconds..."; RETRY_COUNT=$((RETRY_COUNT + 1)); sleep 5; fi; done + - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Error: Mock nodes failed to start after multiple attempts."; exit 1; fi - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: @@ -242,7 +130,6 @@ unit_test service: - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force - rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: -- GitLab From a8eeab3a51aba359d7b8ab1edb2da99858883a2d Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 11:23:51 +0000 Subject: [PATCH 12/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 128 +++++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 18 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 4c76afe71..9e8cb736c 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,14 +49,23 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + + # Context-related - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + + # Device-related - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi + + # Pathcomp-related - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi + + # Service-related - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + script: - docker pull "cockroachdb/cockroach:latest-v22.2" - docker pull "nats:2.9" @@ -65,11 +74,19 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + + # Context preparation - docker volume create crdb - - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 --volume "crdb:/cockroach/cockroach-data" cockroachdb/cockroach:latest-v22.2 start-single-node - - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - > + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 + nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - docker logs nats @@ -78,21 +95,55 @@ unit_test service: - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS - - docker run --name context -d -p 1010:1010 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --env "MB_BACKEND=nats" --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --network=teraflowbridge $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - > + docker run --name context -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "MB_BACKEND=nats" + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CONTEXTSERVICE_SERVICE_HOST - - docker run --name device -d -p 2020:2020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --network=teraflowbridge $CI_REGISTRY_IMAGE/device:$IMAGE_TAG + + # Device preparation + - > + docker run --name device -d -p 2020:2020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $DEVICESERVICE_SERVICE_HOST - - docker run --name pathcomp-backend -d -p 8081:8081 --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG + + # PathComp preparation + - > + docker run --name pathcomp-backend -d -p 8081:8081 + --network=teraflowbridge + $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMP_BACKEND_HOST - sleep 1 - - docker run --name pathcomp-frontend -d -p 10020:10020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" --env "PATHCOMP_BACKEND_PORT=8081" --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG + - > + docker run --name pathcomp-frontend -d -p 10020:10020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" + --env "PATHCOMP_BACKEND_PORT=8081" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - sleep 1 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMPSERVICE_SERVICE_HOST - - docker run --name $IMAGE_NAME -d -p 3030:3030 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + + # Service preparation + - > + docker run --name $IMAGE_NAME -d -p 3030:3030 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" + --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + + # Check status before the tests - sleep 5 - docker ps -a - docker logs context @@ -100,36 +151,76 @@ unit_test service: - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + + # Mock QKD Nodes Deployment - echo "Starting stage: deploy_mock_nodes" - - for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..."; fuser -k $port/tcp; fi; done + - for port in 11111 22222 33333; do + if lsof -i:$port >/dev/null 2>&1; then + echo "Freeing up port $port..."; + fuser -k $port/tcp; + fi + done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit; ./start.sh &; MOCK_NODES_PID=$!; else echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; exit 1; fi + - if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" || exit; + ./start.sh &; + MOCK_NODES_PID=$!; + else + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; + exit 1; + fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && curl -s http://127.0.0.1:22222 > /dev/null && curl -s http://127.0.0.1:33333 > /dev/null; then echo "Mock nodes are up!"; break; else echo "Mock nodes not ready, retrying in 5 seconds..."; RETRY_COUNT=$((RETRY_COUNT + 1)); sleep 5; fi; done - - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Error: Mock nodes failed to start after multiple attempts."; exit 1; fi + - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if curl -s http://127.0.0.1:11111 > /dev/null && \ + curl -s http://127.0.0.1:22222 > /dev/null && \ + curl -s http://127.0.0.1:33333 > /dev/null; then + echo "Mock nodes are up!"; + break; + else + echo "Mock nodes not ready, retrying in 5 seconds..."; + RETRY_COUNT=$((RETRY_COUNT + 1)); + sleep 5; + fi + done + - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Error: Mock nodes failed to start after multiple attempts."; + exit 1; + fi + + # Run the tests + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + + # Run QKD Bootstrap Test - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: + # Check status after the tests - docker ps -a - docker logs context - docker logs device - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME + - docker rm -f $IMAGE_NAME - docker rm -f pathcomp-frontend - docker rm -f pathcomp-backend - docker rm -f device - docker rm -f context - - docker rm -f crdb nats + + - docker rm -f $IMAGE_NAME crdb nats - docker volume rm -f crdb - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force + + rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: @@ -140,10 +231,11 @@ unit_test service: - src/$IMAGE_NAME/tests/*.py - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml + artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy service: -- GitLab From f703f250aea7a9fa240285d73bf903e1aa576f85 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 11:34:51 +0000 Subject: [PATCH 13/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 9e8cb736c..6114a3399 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -156,37 +156,40 @@ unit_test service: - echo "Starting stage: deploy_mock_nodes" - for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..."; - fuser -k $port/tcp; + echo "Freeing up port $port..." + fuser -k $port/tcp fi done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit; - ./start.sh &; - MOCK_NODES_PID=$!; + - | + if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" || exit + ./start.sh & + MOCK_NODES_PID=$! else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found."; - exit 1; + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." + exit 1 fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + - | + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!"; - break; + echo "Mock nodes are up!" + break else - echo "Mock nodes not ready, retrying in 5 seconds..."; - RETRY_COUNT=$((RETRY_COUNT + 1)); - sleep 5; + echo "Mock nodes not ready, retrying in 5 seconds..." + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 5 fi done - - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts."; - exit 1; + - | + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Error: Mock nodes failed to start after multiple attempts." + exit 1 fi # Run the tests -- GitLab From 73d409a4c94e9812dbffb0605814e301555cfd2c Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:00:49 +0000 Subject: [PATCH 14/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 6114a3399..ff96d3b21 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -152,9 +152,10 @@ unit_test service: - docker logs pathcomp-backend - docker logs $IMAGE_NAME - # Mock QKD Nodes Deployment + # Mock QKD Nodes Deployment - echo "Starting stage: deploy_mock_nodes" - - for port in 11111 22222 33333; do + - | + for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..." fuser -k $port/tcp -- GitLab From b7b121654c1de80a01e994b506ff9cb3244b8f21 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:02:33 +0000 Subject: [PATCH 15/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index ff96d3b21..74e56393e 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -154,7 +154,7 @@ unit_test service: # Mock QKD Nodes Deployment - echo "Starting stage: deploy_mock_nodes" - - | + - > for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..." @@ -162,7 +162,7 @@ unit_test service: fi done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - | + - > if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit ./start.sh & @@ -174,7 +174,7 @@ unit_test service: - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - - | + - > while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ @@ -187,7 +187,7 @@ unit_test service: sleep 5 fi done - - | + - > if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Error: Mock nodes failed to start after multiple attempts." exit 1 -- GitLab From 53bcd0e6ff37cec4ab2c2fa50a1e7650dd1fa4d6 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:16:03 +0000 Subject: [PATCH 16/82] Fixed wrong identation --- src/service/.gitlab-ci.yml | 171 ++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 88 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 74e56393e..e0ff82a12 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,22 +49,22 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + - | + # Context-related cleanup + if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - # Context-related - - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + # Device-related cleanup + if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - # Device-related - - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi + # Pathcomp-related cleanup + if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi + if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - # Pathcomp-related - - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi - - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - - # Service-related - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + # Service-related cleanup + if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" @@ -76,93 +76,90 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" # Context preparation - - docker volume create crdb - - > - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 - --volume "crdb:/cockroach/cockroach-data" + - | + docker volume create crdb + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ + --volume "crdb:/cockroach/cockroach-data" \ cockroachdb/cockroach:latest-v22.2 start-single-node - - > - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ nats:2.9 --http_port 8222 --user tfs --pass tfs123 - - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - - docker logs crdb - - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - docker logs nats - - docker ps -a - - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CRDB_ADDRESS - - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $NATS_ADDRESS - - > - docker run --name context -d -p 1010:1010 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" - --network=teraflowbridge + echo "Waiting for initialization..." + while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + docker logs crdb + while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + docker logs nats + docker ps -a + CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $CRDB_ADDRESS + NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $NATS_ADDRESS + + # Context Service Preparation + - | + docker run --name context -d -p 1010:1010 \ + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ + --env "MB_BACKEND=nats" \ + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" \ + --network=teraflowbridge \ $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CONTEXTSERVICE_SERVICE_HOST + CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $CONTEXTSERVICE_SERVICE_HOST # Device preparation - - > - docker run --name device -d -p 2020:2020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --network=teraflowbridge + - | + docker run --name device -d -p 2020:2020 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --network=teraflowbridge \ $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $DEVICESERVICE_SERVICE_HOST + DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $DEVICESERVICE_SERVICE_HOST # PathComp preparation - - > - docker run --name pathcomp-backend -d -p 8081:8081 - --network=teraflowbridge + - | + docker run --name pathcomp-backend -d -p 8081:8081 \ + --network=teraflowbridge \ $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMP_BACKEND_HOST - - sleep 1 - - > - docker run --name pathcomp-frontend -d -p 10020:10020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" - --env "PATHCOMP_BACKEND_PORT=8081" - --network=teraflowbridge + PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $PATHCOMP_BACKEND_HOST + sleep 1 + docker run --name pathcomp-frontend -d -p 10020:10020 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ + --env "PATHCOMP_BACKEND_PORT=8081" \ + --network=teraflowbridge \ $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - - sleep 1 - - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMPSERVICE_SERVICE_HOST + sleep 1 + PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + echo $PATHCOMPSERVICE_SERVICE_HOST # Service preparation - - > - docker run --name $IMAGE_NAME -d -p 3030:3030 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" - --network=teraflowbridge + - | + docker run --name $IMAGE_NAME -d -p 3030:3030 \ + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ + --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" \ + --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" \ + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" \ + --network=teraflowbridge \ $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + sleep 5 + docker ps -a + docker logs context + docker logs device + docker logs pathcomp-frontend + docker logs pathcomp-backend + docker logs $IMAGE_NAME - # Check status before the tests - - sleep 5 - - docker ps -a - - docker logs context - - docker logs device - - docker logs pathcomp-frontend - - docker logs pathcomp-backend - - docker logs $IMAGE_NAME - - # Mock QKD Nodes Deployment - - echo "Starting stage: deploy_mock_nodes" - - > + # Mock QKD Nodes Deployment + - | + echo "Starting stage: deploy_mock_nodes" for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..." fuser -k $port/tcp fi done - - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - - > + MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit ./start.sh & @@ -171,10 +168,9 @@ unit_test service: echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." exit 1 fi - - echo "Waiting for mock nodes to be up..." - - RETRY_COUNT=0 - - MAX_RETRIES=15 - - > + echo "Waiting for mock nodes to be up..." + RETRY_COUNT=0 + MAX_RETRIES=15 while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do if curl -s http://127.0.0.1:11111 > /dev/null && \ curl -s http://127.0.0.1:22222 > /dev/null && \ @@ -187,17 +183,16 @@ unit_test service: sleep 5 fi done - - > if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Error: Mock nodes failed to start after multiple attempts." exit 1 fi # Run the tests - - > - docker exec -i $IMAGE_NAME bash -c + - | + docker exec -i $IMAGE_NAME bash -c \ "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" # Run QKD Bootstrap Test - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" -- GitLab From f013090055e0a32bfe89072b65cf3bab1d6d31e4 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:31:00 +0000 Subject: [PATCH 17/82] Fixed some error --- src/service/.gitlab-ci.yml | 6 ++++-- src/service/Dockerfile | 3 +++ src/tests/tools/mock_qkd_nodes/start.sh | 20 +++++++++----------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index e0ff82a12..0ca6cfa9b 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,6 +49,7 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi + - pip install flask - | # Context-related cleanup if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi @@ -151,8 +152,9 @@ unit_test service: docker logs $IMAGE_NAME # Mock QKD Nodes Deployment + - echo "Starting stage: deploy_mock_nodes" + - pip install flask # Install Flask to ensure it is available - | - echo "Starting stage: deploy_mock_nodes" for port in 11111 22222 33333; do if lsof -i:$port >/dev/null 2>&1; then echo "Freeing up port $port..." @@ -162,7 +164,7 @@ unit_test service: MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit - ./start.sh & + ./start.sh & # The start script that invokes flask commands MOCK_NODES_PID=$! else echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 6f23f0a89..54cc46b96 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -32,6 +32,9 @@ RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel RUN python3 -m pip install --upgrade pip-tools +# Install Flask +RUN python3 -m pip install Flask + # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow diff --git a/src/tests/tools/mock_qkd_nodes/start.sh b/src/tests/tools/mock_qkd_nodes/start.sh index 89797b9c9..cf7f340a3 100755 --- a/src/tests/tools/mock_qkd_nodes/start.sh +++ b/src/tests/tools/mock_qkd_nodes/start.sh @@ -13,30 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/bin/bash cd "$(dirname "$0")" # Function to kill all background processes killbg() { - for p in "${pids[@]}" ; do - kill "$p"; + for p in "${pids[@]}"; do + kill "$p" 2>/dev/null done } +# Trap exit and ensure cleanup of Flask processes trap killbg EXIT pids=() # Set FLASK_APP and run the Flask instances on different ports export FLASK_APP=wsgi -flask run --host 0.0.0.0 --port 11111 & -pids+=($!) - -flask run --host 0.0.0.0 --port 22222 & -pids+=($!) -flask run --host 0.0.0.0 --port 33333 & -pids+=($!) +# Starting Flask instances on different ports +for port in 11111 22222 33333; do + flask run --host 0.0.0.0 --port "$port" & + pids+=($!) + sleep 2 # To avoid conflicts during startup, giving each Flask instance time to initialize +done # Wait for all background processes to finish wait - -- GitLab From 3f755fa625556075f61588c20d2f56829cd31370 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:35:39 +0000 Subject: [PATCH 18/82] Fixed some error --- src/service/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 0ca6cfa9b..ae4ecd7b0 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,7 +49,6 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - pip install flask - | # Context-related cleanup if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi @@ -152,7 +151,8 @@ unit_test service: docker logs $IMAGE_NAME # Mock QKD Nodes Deployment - - echo "Starting stage: deploy_mock_nodes" + - | + echo "Starting stage: deploy_mock_nodes" - pip install flask # Install Flask to ensure it is available - | for port in 11111 22222 33333; do @@ -164,7 +164,7 @@ unit_test service: MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" if [ -d "$MOCK_NODES_DIR" ]; then cd "$MOCK_NODES_DIR" || exit - ./start.sh & # The start script that invokes flask commands + ./start.sh & MOCK_NODES_PID=$! else echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." -- GitLab From 5d6234cdd4f789c51e579fd835a0fb5ca18a944f Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 14:43:52 +0000 Subject: [PATCH 19/82] Added flask dependency --- my_deploy.sh | 12 ++++++------ src/service/Dockerfile | 5 +++-- src/tests/tools/mock_qkd_nodes/start.sh | 8 +++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index 8d2e733d4..2dcd6b6a7 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -80,11 +80,11 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # To manage QKD Apps, "service" requires "qkd_app" to be deployed # before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the # "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. -#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then -# BEFORE="${TFS_COMPONENTS% service*}" -# AFTER="${TFS_COMPONENTS#* service}" -# export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" -#fi +if [[ "$TFS_COMPONENTS" == *"service"* ]]; then + BEFORE="${TFS_COMPONENTS% service*}" + AFTER="${TFS_COMPONENTS#* service}" + export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +fi # Uncomment to activate Load Generator #export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" @@ -140,7 +140,7 @@ export CRDB_DEPLOY_MODE="single" export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. -export CRDB_REDEPLOY="" +export CRDB_REDEPLOY="YES" # ----- NATS ------------------------------------------------------------------- diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 54cc46b96..8056b6fa4 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -32,8 +32,9 @@ RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel RUN python3 -m pip install --upgrade pip-tools -# Install Flask -RUN python3 -m pip install Flask +# Install Flask globally +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install flask # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components diff --git a/src/tests/tools/mock_qkd_nodes/start.sh b/src/tests/tools/mock_qkd_nodes/start.sh index cf7f340a3..7dfd2372d 100755 --- a/src/tests/tools/mock_qkd_nodes/start.sh +++ b/src/tests/tools/mock_qkd_nodes/start.sh @@ -15,14 +15,16 @@ cd "$(dirname "$0")" +# Ensure the local bin directory is in the PATH +export PATH=$PATH:/home/gitlab-runner/.local/bin + # Function to kill all background processes killbg() { - for p in "${pids[@]}"; do - kill "$p" 2>/dev/null + for p in "${pids[@]}" ; do + kill "$p" 2>/dev/null; done } -# Trap exit and ensure cleanup of Flask processes trap killbg EXIT pids=() -- GitLab From b29b527ee31713afc0a9f1dfc763e478823683db Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 21 Oct 2024 15:23:17 +0000 Subject: [PATCH 20/82] Fixed way to deploy mocked qkds --- src/device/.gitlab-ci.yml | 73 ++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 5337a5800..cc5f4c01c 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -44,29 +44,52 @@ build_device: - .gitlab-ci.yml # Deploy mock QKD nodes -deploy_mock_qkd_nodes: +prepare_mock_qkd_nodes: stage: prepare + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge network is already created"; else docker network create --driver=bridge teraflowbridge; fi + - | + # Context-related cleanup + if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + script: - - echo "Starting mock QKD nodes deployment..." - - > - if docker network list | grep -q teraflowbridge; then - echo "teraflowbridge network is already created"; + - docker volume create crdb + - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node + - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + - MOCK_NODES_DIR="$CI_PROJECT_DIR/controller/src/tests/tools/mock_qkd_nodes" + - | + if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" && ./start.sh & + MOCK_NODES_PID=$! else - docker network create -d bridge teraflowbridge; + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; fi + - echo "Waiting for mock nodes to be up..." + - RETRY_COUNT=0 + - MAX_RETRIES=15 - | - for port in 11111 22222 33333; do - if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..."; - fuser -k $port/tcp; + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if curl -s http://127.0.0.1:11111 > /dev/null && \ + curl -s http://127.0.0.1:22222 > /dev/null && \ + curl -s http://127.0.0.1:33333 > /dev/null; then + echo "Mock nodes are up!" + break + else + echo "Mock nodes not ready, retrying in 5 seconds..." + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 5 fi done - - MOCK_NODES_DIR="controller/src/tests/tools/mock_qkd_nodes" - | - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" && ./start.sh & - else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Error: Mock nodes failed to start after multiple attempts." + exit 1 fi rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' @@ -88,17 +111,15 @@ unit_test_device: stage: unit_test needs: - build_device - - deploy_mock_qkd_nodes + - prepare_mock_qkd_nodes before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME --network="host" -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" -e PYTHONPATH="/var/teraflow:/var/teraflow/device:/var/teraflow/tests/tools/mock_qkd_nodes:/var/teraflow/tests" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run --name $IMAGE_NAME --network=teraflowbridge -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" -e PYTHONPATH="/var/teraflow:/var/teraflow/device:/var/teraflow/tests/tools/mock_qkd_nodes:/var/teraflow/tests" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_mock_connectivity.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" @@ -107,7 +128,11 @@ unit_test_device: coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME + - docker rm -f pathcomp-frontend pathcomp-backend device context crdb nats + - docker volume rm -f crdb - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' @@ -127,25 +152,19 @@ unit_test_device: junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml ## Deployment of the service in Kubernetes Cluster -#deploy device: +#deploy_device: # variables: # IMAGE_NAME: 'device' # name of the microservice # IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) # stage: deploy # needs: -# - unit test device -# # - integ_test execute +# - unit_test_device # script: # - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' # - kubectl version # - kubectl get all # - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" # - kubectl get all -# # environment: -# # name: test -# # url: https://example.com -# # kubernetes: -# # namespace: test # rules: # - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' # when: manual -- GitLab From dadecb4f8bd695ebd8fc4f779ca1b487c5b10d9f Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 28 Oct 2024 16:04:31 +0000 Subject: [PATCH 21/82] Added qkd app tests --- src/qkd_app/.gitlab-ci.yml | 44 +++++++++++++++++++ .../tests}/test_create_apps.py | 2 +- .../tests}/test_set_new_configuration.py | 2 +- src/tests/tools/mock_qkd_nodes/wsgi.py | 3 ++ 4 files changed, 49 insertions(+), 2 deletions(-) rename src/{device/tests/qkd/unit => qkd_app/tests}/test_create_apps.py (98%) rename src/{device/tests/qkd/unit => qkd_app/tests}/test_set_new_configuration.py (98%) diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml index 85ee2b5e0..ec07719aa 100644 --- a/src/qkd_app/.gitlab-ci.yml +++ b/src/qkd_app/.gitlab-ci.yml @@ -58,6 +58,50 @@ unit_test app: - docker logs $IMAGE_NAME - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + + # Mock QKD Nodes Deployment + - | + echo "Starting stage: deploy_mock_nodes" + - pip install flask # Install Flask to ensure it is available + - | + for port in 11111 22222 33333; do + if lsof -i:$port >/dev/null 2>&1; then + echo "Freeing up port $port..." + fuser -k $port/tcp + fi + done + MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" + if [ -d "$MOCK_NODES_DIR" ]; then + cd "$MOCK_NODES_DIR" || exit + ./start.sh & + MOCK_NODES_PID=$! + else + echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." + exit 1 + fi + echo "Waiting for mock nodes to be up..." + RETRY_COUNT=0 + MAX_RETRIES=15 + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if curl -s http://127.0.0.1:11111 > /dev/null && \ + curl -s http://127.0.0.1:22222 > /dev/null && \ + curl -s http://127.0.0.1:33333 > /dev/null; then + echo "Mock nodes are up!" + break + else + echo "Mock nodes not ready, retrying in 5 seconds..." + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 5 + fi + done + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Error: Mock nodes failed to start after multiple attempts." + exit 1 + fi + + # Run additional QKD unit tests + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_create_apps.py" + - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_Set_new_configuration.py" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME diff --git a/src/device/tests/qkd/unit/test_create_apps.py b/src/qkd_app/tests/test_create_apps.py similarity index 98% rename from src/device/tests/qkd/unit/test_create_apps.py rename to src/qkd_app/tests/test_create_apps.py index 4724e5c4a..4d05c0e93 100644 --- a/src/device/tests/qkd/unit/test_create_apps.py +++ b/src/qkd_app/tests/test_create_apps.py @@ -14,7 +14,7 @@ import requests -QKD_ADDRESS = '10.0.2.10' +QKD_ADDRESS = '127.0.0.1' QKD_URL = 'http://{:s}/qkd_app/create_qkd_app'.format(QKD_ADDRESS) QKD_REQUEST_1 = { diff --git a/src/device/tests/qkd/unit/test_set_new_configuration.py b/src/qkd_app/tests/test_set_new_configuration.py similarity index 98% rename from src/device/tests/qkd/unit/test_set_new_configuration.py rename to src/qkd_app/tests/test_set_new_configuration.py index 438e46d74..a042a14df 100644 --- a/src/device/tests/qkd/unit/test_set_new_configuration.py +++ b/src/qkd_app/tests/test_set_new_configuration.py @@ -53,7 +53,7 @@ def create_qkd_app(driver, qkdn_id, backing_qkdl_id, client_app_id=None): print(f"Sending payload to {driver.address}: {app_payload}") # Send POST request to create the application - response = requests.post(f'http://{driver.address}/app/create_qkd_app', json=app_payload) + response = requests.post(f'http://{driver.address}/qkd_app/create_qkd_app', json=app_payload) # Check if the request was successful (HTTP 2xx) response.raise_for_status() diff --git a/src/tests/tools/mock_qkd_nodes/wsgi.py b/src/tests/tools/mock_qkd_nodes/wsgi.py index 2a3d48adb..28ba78260 100644 --- a/src/tests/tools/mock_qkd_nodes/wsgi.py +++ b/src/tests/tools/mock_qkd_nodes/wsgi.py @@ -183,6 +183,9 @@ def get_side_effect(url): module = header_splitted[0] assert(module == 'etsi-qkd-sdn-node') + if ip_port.startswith('127.0.0.1'): + ip_port = ip_port.replace('127.0.0.1', current_ip) + tree = {'qkd_node': nodes[ip_port]['node'].copy()} if len(header_splitted) == 1 or not header_splitted[1]: -- GitLab From b6570b05639ce260a76651b9199dfcec0be6d849 Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Mon, 28 Oct 2024 16:14:34 +0000 Subject: [PATCH 22/82] Added qkd_app to global yml pipeline --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e00a85167..22e5a4efe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ include: #- local: '/src/context/.gitlab-ci.yml' - local: '/src/device/.gitlab-ci.yml' - local: '/src/service/.gitlab-ci.yml' + - local: '/src/qkd_app/.gitlab-ci.yml' #- local: '/src/dbscanserving/.gitlab-ci.yml' #- local: '/src/opticalattackmitigator/.gitlab-ci.yml' #- local: '/src/opticalattackdetector/.gitlab-ci.yml' -- GitLab From 431825f0a75d47638afa39c3d140bd7b2693219b Mon Sep 17 00:00:00 2001 From: Luis Carlos Lopes Silva Date: Mon, 10 Feb 2025 18:10:58 +0000 Subject: [PATCH 23/82] Created qkd_end2end folder --- src/tests/qkd_end2end/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/tests/qkd_end2end/.gitkeep diff --git a/src/tests/qkd_end2end/.gitkeep b/src/tests/qkd_end2end/.gitkeep new file mode 100644 index 000000000..e69de29bb -- GitLab From 06efb9670c48e7d72104fc61edfd674501d05860 Mon Sep 17 00:00:00 2001 From: Luis Carlos Lopes Silva Date: Mon, 10 Feb 2025 18:13:21 +0000 Subject: [PATCH 24/82] Added the .gitlab-ci.yml to qkd_end2end --- src/tests/qkd_end2end/.gitlab-ci.yml | 119 +++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 src/tests/qkd_end2end/.gitlab-ci.yml diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml new file mode 100644 index 000000000..fd1c65b84 --- /dev/null +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -0,0 +1,119 @@ +# Build, tag, and push the Docker image to the GitLab Docker registry +build qkd_end2end: + variables: + TEST_NAME: 'qkd_end2end' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test qkd_end2end: + variables: + TEST_NAME: 'qkd_end2end' + stage: end2end_test + # Needs to run after build stage + needs: + - build qkd_end2end + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker rm -f crdb nats mock_qkd context device pathcomp-frontend pathcomp-backend service qkd_end2end + - docker network rm teraflowbridge + + script: + # Create Docker network for inter-container communication + - docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254 teraflowbridge + + # Pull necessary images + - docker pull "$CI_REGISTRY_IMAGE/context:latest" + - docker pull "$CI_REGISTRY_IMAGE/device:latest" + - docker pull "$CI_REGISTRY_IMAGE/service:latest" + - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:latest" + - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:latest" + - docker pull "$CI_REGISTRY_IMAGE/qkd_app:latest" + - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + + # Deploy CockroachDB (crdb) and NATS + - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node --insecure + - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 + + # Wait for CockroachDB and NATS to initialize + - echo "Waiting for CockroachDB to be ready..." + - while ! docker logs crdb 2>&1 | grep -q 'CockroachDB node starting'; do sleep 1; done + - docker logs crdb + - echo "Waiting for NATS to be ready..." + - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + - docker logs nats + + # Start mock QKD nodes + - docker run --name mock_qkd -d --network=teraflowbridge -v "$PWD/src/tests/tools/mock_qkd_nodes:/app" python:3.9-slim bash -c "cd /app && ./start.sh" + + # Wait for mock QKD nodes to initialize + - echo "Waiting for mock QKD nodes to be ready..." + - sleep 10 + + # Deploy TeraFlowSDN services + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + + # Deploy Context Service + - docker run --name context -d -p 1010:1010 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --env "MB_BACKEND=nats" --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --network=teraflowbridge $CI_REGISTRY_IMAGE/context:latest + - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + + # Deploy Device Service + - docker run --name device -d -p 2020:2020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --network=teraflowbridge $CI_REGISTRY_IMAGE/device:latest + - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + + # Deploy PathComp Services (frontend and backend) + - docker run --name pathcomp-backend -d -p 8081:8081 --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-backend:latest + - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - docker run --name pathcomp-frontend -d -p 10020:10020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" --env "PATHCOMP_BACKEND_PORT=8081" --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-frontend:latest + - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + + # Deploy Service Component + - docker run --name service -d -p 3030:3030 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" --volume "$PWD/src/service/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/service:latest + + # Wait for services to initialize + - sleep 10 + + # Run end-to-end tests for QKD application + - docker run --name ${TEST_NAME} -t --network=teraflowbridge --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + + after_script: + # Dump logs for TeraFlowSDN components + - docker logs context + - docker logs device + - docker logs pathcomp-frontend + - docker logs pathcomp-backend + - docker logs service + + # Dump logs for QKD mock nodes + - docker logs mock_qkd + + # Clean up + - docker rm -f context device pathcomp-frontend pathcomp-backend service mock_qkd crdb nats + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report_*.xml -- GitLab From 6705ddb0ece0c470f918e8734077ffb3bfb36ea6 Mon Sep 17 00:00:00 2001 From: Luis Carlos Lopes Silva Date: Tue, 11 Feb 2025 15:47:45 +0000 Subject: [PATCH 25/82] Delete .gitkeep --- src/tests/qkd_end2end/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/tests/qkd_end2end/.gitkeep diff --git a/src/tests/qkd_end2end/.gitkeep b/src/tests/qkd_end2end/.gitkeep deleted file mode 100644 index e69de29bb..000000000 -- GitLab From 86f1c34b4b325d759b88e5bbc30ce5c4793a47b1 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 3 Jul 2025 16:52:45 +0000 Subject: [PATCH 26/82] Mock QKD Node: - generalizing configuration of Mock QKD Node - reorganizing module and code to enable dockerization --- .../tests/qkd/unit/test_qkd_compliance.py | 2 +- src/tests/.gitlab-ci.yml | 24 +- src/tests/tools/mock_qkd_node/.gitlab-ci.yml | 39 ++ src/tests/tools/mock_qkd_node/Dockerfile | 58 +++ src/tests/tools/mock_qkd_node/README.md | 33 ++ .../YangValidator.py | 2 - src/tests/tools/mock_qkd_node/__init__.py | 14 + src/tests/tools/mock_qkd_node/build.sh | 21 ++ .../tools/mock_qkd_node/data/database-01.json | 20 ++ .../tools/mock_qkd_node/data/database-02.json | 25 ++ .../tools/mock_qkd_node/data/database-03.json | 20 ++ src/tests/tools/mock_qkd_node/deploy.sh | 17 + .../tools/mock_qkd_node/mock-qkd-node-01.yaml | 64 ++++ src/tests/tools/mock_qkd_node/requirements.in | 19 + src/tests/tools/mock_qkd_node/run.sh | 19 + .../start.sh | 0 src/tests/tools/mock_qkd_node/tests.py | 43 +++ src/tests/tools/mock_qkd_node/wsgi.py | 151 ++++++++ .../yang/etsi-qkd-node-types.yang | 0 .../yang/etsi-qkd-sdn-node.yang | 0 .../yang/ietf-inet-types.yang | 0 .../yang/ietf-yang-types.yang | 0 src/tests/tools/mock_qkd_nodes/wsgi.py | 334 ------------------ 23 files changed, 557 insertions(+), 348 deletions(-) create mode 100644 src/tests/tools/mock_qkd_node/.gitlab-ci.yml create mode 100644 src/tests/tools/mock_qkd_node/Dockerfile create mode 100644 src/tests/tools/mock_qkd_node/README.md rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/YangValidator.py (99%) create mode 100644 src/tests/tools/mock_qkd_node/__init__.py create mode 100755 src/tests/tools/mock_qkd_node/build.sh create mode 100644 src/tests/tools/mock_qkd_node/data/database-01.json create mode 100644 src/tests/tools/mock_qkd_node/data/database-02.json create mode 100644 src/tests/tools/mock_qkd_node/data/database-03.json create mode 100755 src/tests/tools/mock_qkd_node/deploy.sh create mode 100644 src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml create mode 100644 src/tests/tools/mock_qkd_node/requirements.in create mode 100755 src/tests/tools/mock_qkd_node/run.sh rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/start.sh (100%) create mode 100644 src/tests/tools/mock_qkd_node/tests.py create mode 100644 src/tests/tools/mock_qkd_node/wsgi.py rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/yang/etsi-qkd-node-types.yang (100%) rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/yang/etsi-qkd-sdn-node.yang (100%) rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/yang/ietf-inet-types.yang (100%) rename src/tests/tools/{mock_qkd_nodes => mock_qkd_node}/yang/ietf-yang-types.yang (100%) delete mode 100644 src/tests/tools/mock_qkd_nodes/wsgi.py diff --git a/src/device/tests/qkd/unit/test_qkd_compliance.py b/src/device/tests/qkd/unit/test_qkd_compliance.py index 292268c44..4dec4ab6a 100644 --- a/src/device/tests/qkd/unit/test_qkd_compliance.py +++ b/src/device/tests/qkd/unit/test_qkd_compliance.py @@ -16,7 +16,7 @@ import pytest import requests from requests.exceptions import HTTPError -from tests.tools.mock_qkd_nodes.YangValidator import YangValidator +from tests.tools.mock_qkd_node.YangValidator import YangValidator def test_compliance_with_yang_models(): validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index dfccc4726..5a0dd6883 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -14,16 +14,18 @@ # include the individual .gitlab-ci.yml of each end-to-end integration test include: - - local: '/src/tests/ofc22/.gitlab-ci.yml' - #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' - - local: '/src/tests/ecoc22/.gitlab-ci.yml' - #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' - #- local: '/src/tests/ofc23/.gitlab-ci.yml' - - local: '/src/tests/ofc24/.gitlab-ci.yml' - - local: '/src/tests/eucnc24/.gitlab-ci.yml' - #- local: '/src/tests/ofc25-camara-agg-net-controller/.gitlab-ci.yml' - #- local: '/src/tests/ofc25-camara-e2e-controller/.gitlab-ci.yml' - #- local: '/src/tests/ofc25/.gitlab-ci.yml' - #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' +# - local: '/src/tests/ofc22/.gitlab-ci.yml' +# #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' +# - local: '/src/tests/ecoc22/.gitlab-ci.yml' +# #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' +# #- local: '/src/tests/ofc23/.gitlab-ci.yml' +# - local: '/src/tests/ofc24/.gitlab-ci.yml' +# - local: '/src/tests/eucnc24/.gitlab-ci.yml' +# #- local: '/src/tests/ofc25-camara-agg-net-controller/.gitlab-ci.yml' +# #- local: '/src/tests/ofc25-camara-e2e-controller/.gitlab-ci.yml' +# #- local: '/src/tests/ofc25/.gitlab-ci.yml' +# #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' + - local: '/src/tests/qkd_end2end/.gitlab-ci.yml' - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' + - local: '/src/tests/tools/mock_qkd_node/.gitlab-ci.yml' diff --git a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml new file mode 100644 index 000000000..787f27072 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml @@ -0,0 +1,39 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build mock_qkd_node: + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$CI_REGISTRY_IMAGE/mock_qkd_node:test" -f ./src/tests/tools/mock_qkd_node/Dockerfile . + - docker push "$CI_REGISTRY_IMAGE/mock_qkd_node:test" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/src/tests/tools/mock_qkd_node/**/*.{py,in,yml,yaml} + - src/src/tests/tools/mock_qkd_node/Dockerfile + - src/device/**/*.{py,in,yml} + - src/device/Dockerfile + - src/device/tests/*.py + - src/qkd_app/**/*.{py,in,yml} + - src/qkd_app/Dockerfile + - src/qkd_app/tests/*.py + - .gitlab-ci.yml diff --git a/src/tests/tools/mock_qkd_node/Dockerfile b/src/tests/tools/mock_qkd_node/Dockerfile new file mode 100644 index 000000000..510de0831 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install g++ git build-essential cmake libpcre2-dev python3-dev python3-cffi && \ + rm -rf /var/lib/apt/lists/* + +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +WORKDIR /var/libyang +RUN git fetch +RUN git checkout v2.1.148 +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Create component sub-folders, and copy content +RUN mkdir -p /var/teraflow/mock_qkd_node/{data,yang} +WORKDIR /var/teraflow/mock_qkd_node +COPY yang/. yang/ +COPY requirements.in requirements.in +COPY wsgi.py wsgi.py +COPY YangValidator.py YangValidator.py + +# Get specific Python packages +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Start the service +ENTRYPOINT ["gunicorn", "--workers", "1", "--bind", "0.0.0.0:8080", "wsgi:app"] diff --git a/src/tests/tools/mock_qkd_node/README.md b/src/tests/tools/mock_qkd_node/README.md new file mode 100644 index 000000000..52aa2922d --- /dev/null +++ b/src/tests/tools/mock_qkd_node/README.md @@ -0,0 +1,33 @@ +# Mock IETF ACTN SDN Controller + +This REST server implements very basic support for the following YANG data models: +- IETF YANG Data Model for Transport Network Client Signals (draft-ietf-ccamp-client-signal-yang-10) + - Ref: https://datatracker.ietf.org/doc/draft-ietf-ccamp-client-signal-yang/ +- IETF YANG Data Model for Traffic Engineering Tunnels, Label Switched Paths and Interfaces (draft-ietf-teas-yang-te-34) + - Ref: https://datatracker.ietf.org/doc/draft-ietf-teas-yang-te/ + +The aim of this server is to enable testing the IetfActnDeviceDriver and the IetfActnServiceHandler. + + +## 1. Install requirements for the Mock IETF ACTN SDN controller +__NOTE__: if you run the Mock IETF ACTN SDN controller from the PyEnv used for developing on the TeraFlowSDN +framework and you followed the official steps in +[Development Guide > Configure Environment > Python](https://labs.etsi.org/rep/tfs/controller/-/wikis/2.-Development-Guide/2.1.-Configure-Environment/2.1.1.-Python), +all the requirements are already in place. Install them only if you execute it in a separate/standalone environment. + +Install the required dependencies as follows: +```bash +pip install -r src/tests/tools/mock_ietf_actn_sdn_ctrl/requirements.in +``` + +Run the Mock IETF ACTN SDN Controller as follows: +```bash +python src/tests/tools/mock_ietf_actn_sdn_ctrl/MockIetfActnSdnCtrl.py +``` + + +## 2. Run the Mock IETF ACTN SDN controller +Run the Mock IETF ACTN SDN Controller as follows: +```bash +python src/tests/tools/mock_ietf_actn_sdn_ctrl/MockIetfActnSdnCtrl.py +``` diff --git a/src/tests/tools/mock_qkd_nodes/YangValidator.py b/src/tests/tools/mock_qkd_node/YangValidator.py similarity index 99% rename from src/tests/tools/mock_qkd_nodes/YangValidator.py rename to src/tests/tools/mock_qkd_node/YangValidator.py index 4948239ed..6214074c8 100644 --- a/src/tests/tools/mock_qkd_nodes/YangValidator.py +++ b/src/tests/tools/mock_qkd_node/YangValidator.py @@ -26,8 +26,6 @@ class YangValidator: for mod in mods: mod.feature_enable_all() - - def parse_to_dict(self, message : Dict) -> Dict: dnode : Optional[libyang.DNode] = self._yang_module.parse_data_dict( diff --git a/src/tests/tools/mock_qkd_node/__init__.py b/src/tests/tools/mock_qkd_node/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/tests/tools/mock_qkd_node/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/tools/mock_qkd_node/build.sh b/src/tests/tools/mock_qkd_node/build.sh new file mode 100755 index 000000000..8360d93b9 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make folder containing the script the root folder for its execution +cd $(dirname $0) + +docker build -t mock-qkd-node:test -f Dockerfile . +docker tag mock-qkd-node:test localhost:32000/tfs/mock-qkd-node:test +docker push localhost:32000/tfs/mock-qkd-node:test diff --git a/src/tests/tools/mock_qkd_node/data/database-01.json b/src/tests/tools/mock_qkd_node/data/database-01.json new file mode 100644 index 000000000..1f78f7d6d --- /dev/null +++ b/src/tests/tools/mock_qkd_node/data/database-01.json @@ -0,0 +1,20 @@ +{ + "node": {"qkdn_id": "00000001-0000-0000-0000-000000000000"}, + "qkdn_capabilities": {}, + "qkd_applications": {"qkd_app": []}, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "100", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "101", + "qkdi_att_point": {"device": current_ip, "port": "1001"}, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": {"qkd_link": []} +} diff --git a/src/tests/tools/mock_qkd_node/data/database-02.json b/src/tests/tools/mock_qkd_node/data/database-02.json new file mode 100644 index 000000000..4e3a00724 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/data/database-02.json @@ -0,0 +1,25 @@ +{ + "node": {"qkdn_id": "00000002-0000-0000-0000-000000000000"}, + "qkdn_capabilities": {}, + "qkd_applications": {"qkd_app": []}, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "200", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "201", + "qkdi_att_point": {"device": current_ip, "port": "2001"}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "202", + "qkdi_att_point": {"device": current_ip, "port": "2002"}, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": {"qkd_link": []} +} diff --git a/src/tests/tools/mock_qkd_node/data/database-03.json b/src/tests/tools/mock_qkd_node/data/database-03.json new file mode 100644 index 000000000..b6dbb7aa5 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/data/database-03.json @@ -0,0 +1,20 @@ +{ + "node": {"qkdn_id": "00000003-0000-0000-0000-000000000000"}, + "qkdn_capabilities": {}, + "qkd_applications": {"qkd_app": []}, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "300", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "301", + "qkdi_att_point": {"device": current_ip, "port": "3001"}, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": {"qkd_link": []} +} diff --git a/src/tests/tools/mock_qkd_node/deploy.sh b/src/tests/tools/mock_qkd_node/deploy.sh new file mode 100755 index 000000000..5453efc8c --- /dev/null +++ b/src/tests/tools/mock_qkd_node/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl delete namespace mocks +kubectl --namespace mocks apply -f mock-qkd-node.yaml diff --git a/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml b/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml new file mode 100644 index 000000000..ad718fc4e --- /dev/null +++ b/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml @@ -0,0 +1,64 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Namespace +apiVersion: v1 +metadata: + name: mocks +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-qkd-node-01 +spec: + selector: + matchLabels: + app: mock-qkd-node-01 + replicas: 1 + template: + metadata: + annotations: + config.linkerd.io/skip-inbound-ports: "8443" + labels: + app: mock-qkd-node-01 + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: localhost:32000/tfs/mock-qkd-node:test + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8443 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 700m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: mock-qkd-node-01 + labels: + app: mock-qkd-node-01 +spec: + type: ClusterIP + selector: + app: mock-qkd-node-01 + ports: + - name: https + port: 8443 + targetPort: 8443 diff --git a/src/tests/tools/mock_qkd_node/requirements.in b/src/tests/tools/mock_qkd_node/requirements.in new file mode 100644 index 000000000..cb2f89992 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/requirements.in @@ -0,0 +1,19 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flask==2.1.3 +Flask-HTTPAuth==4.5.0 +Flask-RESTful==0.3.9 +gunicorn==23.0.0 +werkzeug==2.3.7 diff --git a/src/tests/tools/mock_qkd_node/run.sh b/src/tests/tools/mock_qkd_node/run.sh new file mode 100755 index 000000000..ecef1d5d5 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/run.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make folder containing the script the root folder for its execution +cd $(dirname $0) + +python MockIetfActnSdnCtrl.py diff --git a/src/tests/tools/mock_qkd_nodes/start.sh b/src/tests/tools/mock_qkd_node/start.sh similarity index 100% rename from src/tests/tools/mock_qkd_nodes/start.sh rename to src/tests/tools/mock_qkd_node/start.sh diff --git a/src/tests/tools/mock_qkd_node/tests.py b/src/tests/tools/mock_qkd_node/tests.py new file mode 100644 index 000000000..b112595dd --- /dev/null +++ b/src/tests/tools/mock_qkd_node/tests.py @@ -0,0 +1,43 @@ +import json +from mock import requests +import pyangbind.lib.pybindJSON as enc +from pyangbind.lib.serialise import pybindJSONDecoder as dec +from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node + +module = etsi_qkd_sdn_node() +url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:qkd_node/' + +# Get node all info +z = requests.get(url).json() +var = dec.load_json(z, None, None, obj=module) +print(enc.dumps(var)) + +# Reset module variable because it is already filled +module = etsi_qkd_sdn_node() + +# Get node basic info +node = module.qkd_node +z = requests.get(url).json() +var = dec.load_json(z, None, None, obj=node) +print(enc.dumps(var)) + +# Get all apps +apps = node.qkd_applications +z = requests.get(url + 'qkd_applications').json() +var = dec.load_json(z, None, None, obj=apps) +print(enc.dumps(var)) + +# Edit app 0 +app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] +app.client_app_id = 'id_0' +requests.put(url + 'qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) + +# Create app 1 +app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') +requests.post(url + 'qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) + +# Get all apps +apps = node.qkd_applications +z = requests.get(url + 'qkd_applications').json() +var = dec.load_json(z, None, None, obj=apps) +print(enc.dumps(var)) diff --git a/src/tests/tools/mock_qkd_node/wsgi.py b/src/tests/tools/mock_qkd_node/wsgi.py new file mode 100644 index 000000000..e56e3cb2c --- /dev/null +++ b/src/tests/tools/mock_qkd_node/wsgi.py @@ -0,0 +1,151 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, os +from flask import Flask, request +from YangValidator import YangValidator + +DATA_FILE_PATH = os.environ.get('DATA_FILE_PATH') +if DATA_FILE_PATH is None: + raise Exception('DataFile({:s}) is empty'.format(str(DATA_FILE_PATH))) + +# TODO: if file not found, raise exception + +with open(DATA_FILE_PATH, mode='r', encoding='UTF-8') as fp: + data = json.load(fp) + +yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) + +app = Flask(__name__) + +def get_side_effect(url): + steps = url.lstrip('https://').lstrip('http://').rstrip('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") + + ip_port, _, _, header, *steps = parts + + header_splitted = header.split(':') + + module = header_splitted[0] + assert(module == 'etsi-qkd-sdn-node') + + if ip_port.startswith('127.0.0.1'): + ip_port = ip_port.replace('127.0.0.1', current_ip) + + tree = {'qkd_node': nodes[ip_port]['node'].copy()} + + if len(header_splitted) == 1 or not header_splitted[1]: + value = nodes[ip_port].copy() + value.pop('node') + tree['qkd_node'].update(value) + + return tree, tree + + root = header_splitted[1] + assert(root == 'qkd_node') + + if not steps: + return tree, tree + + endpoint, *steps = steps + + value = nodes[ip_port][endpoint] + + if not steps: + return_value = {endpoint: value} + tree['qkd_node'].update(return_value) + + return return_value, tree + + raise Exception('Url too long') + +def edit(from_dict, to_dict, create): + for key, value in from_dict.items(): + if isinstance(value, dict): + if key not in to_dict and create: + to_dict[key] = {} + edit(from_dict[key], to_dict[key], create) + elif isinstance(value, list): + to_dict[key].extend(value) + else: + to_dict[key] = value + +def edit_side_effect(url, json, create): + steps = url.lstrip('https://').lstrip('http://').rstrip('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") + + ip_port, _, _, header, *steps = parts + + module, root = header.split(':') + + assert(module == 'etsi-qkd-sdn-node') + assert(root == 'qkd_node') + + if not steps: + edit(json, nodes[ip_port]['node'], create) + return + + endpoint, *steps = steps + + if not steps: + edit(json[endpoint], nodes[ip_port][endpoint], create) + return + + raise Exception('Url too long') + +@app.get('/', defaults={'path': ''}) +@app.get("/") +@app.get('/') +def get(path): + try: + msg, msg_validate = get_side_effect(request.base_url) + print(msg_validate) + yang_validator.parse_to_dict(msg_validate) + return msg + except ValueError as e: + return {'error': str(e)}, 400 + +@app.post('/', defaults={'path': ''}) +@app.post("/") +@app.post('/') +def post(path): + success = True + reason = '' + try: + edit_side_effect(request.base_url, request.json, True) + except Exception as e: + reason = str(e) + success = False + return {'success': success, 'reason': reason} + +@app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) +@app.route("/", methods=['PUT', 'PATCH']) +@app.route('/', methods=['PUT', 'PATCH']) +def patch(path): + success = True + reason = '' + try: + edit_side_effect(request.base_url, request.json, False) + except Exception as e: + reason = str(e) + success = False + return {'success': success, 'reason': reason} diff --git a/src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-node-types.yang b/src/tests/tools/mock_qkd_node/yang/etsi-qkd-node-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-node-types.yang rename to src/tests/tools/mock_qkd_node/yang/etsi-qkd-node-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-sdn-node.yang b/src/tests/tools/mock_qkd_node/yang/etsi-qkd-sdn-node.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-sdn-node.yang rename to src/tests/tools/mock_qkd_node/yang/etsi-qkd-sdn-node.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/ietf-inet-types.yang b/src/tests/tools/mock_qkd_node/yang/ietf-inet-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/ietf-inet-types.yang rename to src/tests/tools/mock_qkd_node/yang/ietf-inet-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/ietf-yang-types.yang b/src/tests/tools/mock_qkd_node/yang/ietf-yang-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/ietf-yang-types.yang rename to src/tests/tools/mock_qkd_node/yang/ietf-yang-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/wsgi.py b/src/tests/tools/mock_qkd_nodes/wsgi.py deleted file mode 100644 index 6476609cd..000000000 --- a/src/tests/tools/mock_qkd_nodes/wsgi.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import socket -from flask import Flask, request -from YangValidator import YangValidator - -app = Flask(__name__) - -# Retrieve the IP address of the current machine -current_ip = socket.gethostbyname(socket.gethostname()) - -yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - -# Update IP address with the current machine's IP -nodes = { - f'{current_ip}:11111': {'node': { - 'qkdn_id': '00000001-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000001-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '100', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '101', - 'qkdi_att_point': { - 'device': current_ip, - 'port': '1001' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [] - } - }, - - f'{current_ip}:22222': {'node': { - 'qkdn_id': '00000002-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000002-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '200', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '201', - 'qkdi_att_point': { - 'device': current_ip, - 'port': '2001' - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '202', - 'qkdi_att_point': { - 'device': current_ip, - 'port': '2002' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [] - } - }, - - f'{current_ip}:33333': {'node': { - 'qkdn_id': '00000003-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000003-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '300', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '301', - 'qkdi_att_point': { - 'device': current_ip, - 'port': '3001' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [] - } - } -} - -def get_side_effect(url): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") - - ip_port, _, _, header, *steps = parts - - header_splitted = header.split(':') - - module = header_splitted[0] - assert(module == 'etsi-qkd-sdn-node') - - if ip_port.startswith('127.0.0.1'): - ip_port = ip_port.replace('127.0.0.1', current_ip) - - tree = {'qkd_node': nodes[ip_port]['node'].copy()} - - if len(header_splitted) == 1 or not header_splitted[1]: - value = nodes[ip_port].copy() - value.pop('node') - tree['qkd_node'].update(value) - - return tree, tree - - root = header_splitted[1] - assert(root == 'qkd_node') - - if not steps: - return tree, tree - - endpoint, *steps = steps - - value = nodes[ip_port][endpoint] - - if not steps: - return_value = {endpoint: value} - tree['qkd_node'].update(return_value) - - return return_value, tree - - raise Exception('Url too long') - -def edit(from_dict, to_dict, create): - for key, value in from_dict.items(): - if isinstance(value, dict): - if key not in to_dict and create: - to_dict[key] = {} - edit(from_dict[key], to_dict[key], create) - elif isinstance(value, list): - to_dict[key].extend(value) - else: - to_dict[key] = value - -def edit_side_effect(url, json, create): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") - - ip_port, _, _, header, *steps = parts - - module, root = header.split(':') - - assert(module == 'etsi-qkd-sdn-node') - assert(root == 'qkd_node') - - if not steps: - edit(json, nodes[ip_port]['node'], create) - return - - endpoint, *steps = steps - - if not steps: - edit(json[endpoint], nodes[ip_port][endpoint], create) - return - - raise Exception('Url too long') - -@app.get('/', defaults={'path': ''}) -@app.get("/") -@app.get('/') -def get(path): - try: - msg, msg_validate = get_side_effect(request.base_url) - print(msg_validate) - yang_validator.parse_to_dict(msg_validate) - return msg - except ValueError as e: - return {'error': str(e)}, 400 - -@app.post('/', defaults={'path': ''}) -@app.post("/") -@app.post('/') -def post(path): - success = True - reason = '' - try: - edit_side_effect(request.base_url, request.json, True) - except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} - -@app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) -@app.route("/", methods=['PUT', 'PATCH']) -@app.route('/', methods=['PUT', 'PATCH']) -def patch(path): - success = True - reason = '' - try: - edit_side_effect(request.base_url, request.json, False) - except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} - -# import json -# from mock import requests -# import pyangbind.lib.pybindJSON as enc -# from pyangbind.lib.serialise import pybindJSONDecoder as dec -# from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node - -# module = etsi_qkd_sdn_node() -# url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:' - -# # Get node all info -# z = requests.get(url).json() -# var = dec.load_json(z, None, None, obj=module) -# print(enc.dumps(var)) - -# Reset module variable because it is already filled -# module = etsi_qkd_sdn_node() - -# # Get node basic info -# node = module.qkd_node -# z = requests.get(url + 'qkd_node').json() -# var = dec.load_json(z, None, None, obj=node) -# print(enc.dumps(var)) - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var)) - -# # Edit app 0 -# app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] -# app.client_app_id = 'id_0' -# requests.put(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) - -# # Create app 1 -# app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') -# requests.post(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var)) -- GitLab From f9f3fc73fd30e4b4ce135e7faa1243191077a43d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 3 Jul 2025 16:53:09 +0000 Subject: [PATCH 27/82] CI/CD pipeline - QKD E2E test: - added topology descriptor --- src/tests/qkd_end2end/topology_qkd.json | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/tests/qkd_end2end/topology_qkd.json diff --git a/src/tests/qkd_end2end/topology_qkd.json b/src/tests/qkd_end2end/topology_qkd.json new file mode 100644 index 000000000..aa301baff --- /dev/null +++ b/src/tests/qkd_end2end/topology_qkd.json @@ -0,0 +1,52 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + } + ], + "links": [ + {"link_id": {"link_uuid": {"uuid": "QKD1/10.0.2.10:1001==QKD2/10.0.2.10:2001"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2001==QKD1/10.0.2.10:1001"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2002==QKD3/10.0.2.10:3001"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD3/10.0.2.10:3001==QKD2/10.0.2.10:2002"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}} + ]} + ] +} -- GitLab From fd4277363030895739daede7d48dd74ca357f1ef Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 3 Jul 2025 16:53:28 +0000 Subject: [PATCH 28/82] CI/CD pipeline - QKD E2E test: - enabled DEBUG logger messages --- manifests/contextservice.yaml | 2 +- manifests/deviceservice.yaml | 2 +- manifests/nbiservice.yaml | 4 ++-- manifests/pathcompservice.yaml | 2 +- manifests/qkd_appservice.yaml | 2 +- manifests/serviceservice.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 5592864d6..2bf8b5d86 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -40,7 +40,7 @@ spec: - name: MB_BACKEND value: "nats" - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY value: "FALSE" - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index a366a5041..7c3ded7c0 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:2020"] diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index cead19406..6a07478c0 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -39,9 +39,9 @@ spec: #- containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: FLASK_ENV - value: "production" # change to "development" if developing + value: "development" # change to "development" if developing - name: IETF_NETWORK_RENDERER value: "LIBYANG" envFrom: diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 2db0d41b0..71c7e4cd7 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: ENABLE_FORECASTER value: "NO" readinessProbe: diff --git a/manifests/qkd_appservice.yaml b/manifests/qkd_appservice.yaml index 79cfaf6bd..7641bd3aa 100644 --- a/manifests/qkd_appservice.yaml +++ b/manifests/qkd_appservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: CRDB_DATABASE value: "tfs_qkd_app" envFrom: diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 8262550ef..8615e8879 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:3030"] -- GitLab From cd7e3a752b4bde846decb5bca70e0eb5115e00a9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 3 Jul 2025 16:56:12 +0000 Subject: [PATCH 29/82] CI/CD pipeline - QKD E2E test: - added TFS deploy specs and scripts --- src/tests/qkd_end2end/deploy_specs.sh | 213 ++++++++++++++++++++++++++ src/tests/qkd_end2end/redeploy-tfs.sh | 17 ++ 2 files changed, 230 insertions(+) create mode 100755 src/tests/qkd_end2end/deploy_specs.sh create mode 100755 src/tests/qkd_end2end/redeploy-tfs.sh diff --git a/src/tests/qkd_end2end/deploy_specs.sh b/src/tests/qkd_end2end/deploy_specs.sh new file mode 100755 index 000000000..face2f5b0 --- /dev/null +++ b/src/tests/qkd_end2end/deploy_specs.sh @@ -0,0 +1,213 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +export TFS_COMPONENTS="context device pathcomp service nbi webui" + +# Uncomment to activate Monitoring (old) +#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate Monitoring Framework (new) +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" + +# Uncomment to activate QoS Profiles +#export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" + +# Uncomment to activate BGP-LS Speaker +#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" + +# Uncomment to activate Optical Controller +# To manage optical connections, "service" requires "opticalcontroller" to be deployed +# before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the +# "opticalcontroller" only if "service" is already in TFS_COMPONENTS, and re-export it. +#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then +# BEFORE="${TFS_COMPONENTS% service*}" +# AFTER="${TFS_COMPONENTS#* service}" +# export TFS_COMPONENTS="${BEFORE} opticalcontroller service ${AFTER}" +#fi + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + +# Uncomment to activate VNT Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} vnt_manager" + +# Uncomment to activate DLT and Interdomain +#export TFS_COMPONENTS="${TFS_COMPONENTS} interdomain dlt" +#if [[ "$TFS_COMPONENTS" == *"dlt"* ]]; then +# export KEY_DIRECTORY_PATH="src/dlt/gateway/keys/priv_sk" +# export CERT_DIRECTORY_PATH="src/dlt/gateway/keys/cert.pem" +# export TLS_CERT_PATH="src/dlt/gateway/keys/ca.crt" +#fi + +# Uncomment to activate QKD App +# To manage QKD Apps, "service" requires "qkd_app" to be deployed +# before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the +# "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. +if [[ "$TFS_COMPONENTS" == *"service"* ]]; then + BEFORE="${TFS_COMPONENTS% service*}" + AFTER="${TFS_COMPONENTS#* service}" + export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +fi + +# Uncomment to activate Load Generator +#export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" + + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="YES" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Set NATS installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/nats.sh for additional details +export NATS_DEPLOY_MODE="single" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" + + +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + +# Set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT="9092" + +# Set the flag to YES for redeploying of Apache Kafka +export KFK_REDEPLOY="" diff --git a/src/tests/qkd_end2end/redeploy-tfs.sh b/src/tests/qkd_end2end/redeploy-tfs.sh new file mode 100755 index 000000000..7433ae4a1 --- /dev/null +++ b/src/tests/qkd_end2end/redeploy-tfs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh +./deploy/all.sh -- GitLab From 6bd11386abef6ae86a122d4a79e4ae3f553cf593 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 10 Jul 2025 13:35:10 +0000 Subject: [PATCH 30/82] CI/CD pipeline - QKD E2E test: - Remove unneeded files Factorize and update code --- src/tests/tools/mock_qkd_node/QkdNode.py | 46 +++++++++++++ src/tests/tools/mock_qkd_node/README.md | 49 +++++++------- src/tests/tools/mock_qkd_node/deploy.sh | 17 ----- .../tools/mock_qkd_node/mock-qkd-node-01.yaml | 64 ------------------- src/tests/tools/mock_qkd_node/start.sh | 42 ------------ src/tests/tools/mock_qkd_node/wsgi.py | 53 +++++++++++++-- 6 files changed, 117 insertions(+), 154 deletions(-) create mode 100644 src/tests/tools/mock_qkd_node/QkdNode.py delete mode 100755 src/tests/tools/mock_qkd_node/deploy.sh delete mode 100644 src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml delete mode 100755 src/tests/tools/mock_qkd_node/start.sh diff --git a/src/tests/tools/mock_qkd_node/QkdNode.py b/src/tests/tools/mock_qkd_node/QkdNode.py new file mode 100644 index 000000000..7ab25876e --- /dev/null +++ b/src/tests/tools/mock_qkd_node/QkdNode.py @@ -0,0 +1,46 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# REST-API resource implementing minimal support for "IETF YANG Data Model for Transport Network Client Signals". +# Ref: https://www.ietf.org/archive/id/draft-ietf-ccamp-client-signal-yang-10.html + +import json, os +from flask import jsonify, make_response, request +from flask_restful import Resource + + +DATA_FILE_PATH = os.environ.get('DATA_FILE_PATH') +if DATA_FILE_PATH is None: + raise Exception('DataFile({:s}) is not defined'.format(str(DATA_FILE_PATH))) + +if not os.path.isfile(DATA_FILE_PATH): + raise Exception('DataFile({:s}) not found'.format(str(DATA_FILE_PATH))) + +with open(DATA_FILE_PATH, mode='r', encoding='UTF-8') as fp: + QKD_NODE = json.load(fp) + +class QkdNode(Resource): + def get(self): + return make_response(jsonify(QKD_NODE), 200) + + def post(self): + json_request = request.get_json() + name = json_request["network-slice-services"]["slice-service"][0]["id"] + NETWORK_SLICES[name] = json_request + return make_response(jsonify({}), 201) + + def delete(self, slice_id: str): + slice = NETWORK_SLICES.pop(slice_id, None) + data, status = ({}, 404) if slice is None else (slice, 204) + return make_response(jsonify(data), status) diff --git a/src/tests/tools/mock_qkd_node/README.md b/src/tests/tools/mock_qkd_node/README.md index 52aa2922d..7087fe061 100644 --- a/src/tests/tools/mock_qkd_node/README.md +++ b/src/tests/tools/mock_qkd_node/README.md @@ -1,33 +1,34 @@ -# Mock IETF ACTN SDN Controller +# Mock QKD Node -This REST server implements very basic support for the following YANG data models: -- IETF YANG Data Model for Transport Network Client Signals (draft-ietf-ccamp-client-signal-yang-10) - - Ref: https://datatracker.ietf.org/doc/draft-ietf-ccamp-client-signal-yang/ -- IETF YANG Data Model for Traffic Engineering Tunnels, Label Switched Paths and Interfaces (draft-ietf-teas-yang-te-34) - - Ref: https://datatracker.ietf.org/doc/draft-ietf-teas-yang-te/ +This Mock implements very basic support for the software-defined QKD node information models specified in ETSI GS QKD 015 V2.1.1. -The aim of this server is to enable testing the IetfActnDeviceDriver and the IetfActnServiceHandler. +The aim of this mock is to enable testing the TFS QKD Framework with an emulated data plane. -## 1. Install requirements for the Mock IETF ACTN SDN controller -__NOTE__: if you run the Mock IETF ACTN SDN controller from the PyEnv used for developing on the TeraFlowSDN -framework and you followed the official steps in -[Development Guide > Configure Environment > Python](https://labs.etsi.org/rep/tfs/controller/-/wikis/2.-Development-Guide/2.1.-Configure-Environment/2.1.1.-Python), -all the requirements are already in place. Install them only if you execute it in a separate/standalone environment. - -Install the required dependencies as follows: +## Build the Mock QKD Node Docker image ```bash -pip install -r src/tests/tools/mock_ietf_actn_sdn_ctrl/requirements.in +./build.sh ``` -Run the Mock IETF ACTN SDN Controller as follows: -```bash -python src/tests/tools/mock_ietf_actn_sdn_ctrl/MockIetfActnSdnCtrl.py -``` - - -## 2. Run the Mock IETF ACTN SDN controller -Run the Mock IETF ACTN SDN Controller as follows: +## Run the Mock QKD Node as a container: ```bash -python src/tests/tools/mock_ietf_actn_sdn_ctrl/MockIetfActnSdnCtrl.py +docker network create --driver bridge --subnet=172.254.252.0/24 --gateway=172.254.252.254 tfs-qkd-net-mgmt + +docker run --name qkd-node-01 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.101 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-01.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test + +docker run --name qkd-node-02 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.102 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-02.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test + +docker run --name qkd-node-03 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.103 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-03.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test ``` diff --git a/src/tests/tools/mock_qkd_node/deploy.sh b/src/tests/tools/mock_qkd_node/deploy.sh deleted file mode 100755 index 5453efc8c..000000000 --- a/src/tests/tools/mock_qkd_node/deploy.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kubectl delete namespace mocks -kubectl --namespace mocks apply -f mock-qkd-node.yaml diff --git a/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml b/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml deleted file mode 100644 index ad718fc4e..000000000 --- a/src/tests/tools/mock_qkd_node/mock-qkd-node-01.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Namespace -apiVersion: v1 -metadata: - name: mocks ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mock-qkd-node-01 -spec: - selector: - matchLabels: - app: mock-qkd-node-01 - replicas: 1 - template: - metadata: - annotations: - config.linkerd.io/skip-inbound-ports: "8443" - labels: - app: mock-qkd-node-01 - spec: - terminationGracePeriodSeconds: 5 - containers: - - name: server - image: localhost:32000/tfs/mock-qkd-node:test - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8443 - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 700m - memory: 1024Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: mock-qkd-node-01 - labels: - app: mock-qkd-node-01 -spec: - type: ClusterIP - selector: - app: mock-qkd-node-01 - ports: - - name: https - port: 8443 - targetPort: 8443 diff --git a/src/tests/tools/mock_qkd_node/start.sh b/src/tests/tools/mock_qkd_node/start.sh deleted file mode 100755 index bd5758ee2..000000000 --- a/src/tests/tools/mock_qkd_node/start.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd "$(dirname "$0")" - -# Ensure the local bin directory is in the PATH -export PATH=$PATH:/home/gitlab-runner/.local/bin - -# Function to kill all background processes -killbg() { - for p in "${pids[@]}" ; do - kill "$p" 2>/dev/null; - done -} - -trap killbg EXIT -pids=() - -# Set FLASK_APP and run the Flask instances on different ports -export FLASK_APP=wsgi - -# Starting Flask instances on different ports -for port in 11111 22222 33333; do - flask run --host 0.0.0.0 --port "$port" & - pids+=($!) - sleep 2 # To avoid conflicts during startup, giving each Flask instance time to initialize -done - -# Wait for all background processes to finish -wait diff --git a/src/tests/tools/mock_qkd_node/wsgi.py b/src/tests/tools/mock_qkd_node/wsgi.py index e56e3cb2c..184b9c006 100644 --- a/src/tests/tools/mock_qkd_node/wsgi.py +++ b/src/tests/tools/mock_qkd_node/wsgi.py @@ -12,22 +12,61 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, os +import functools, json, logging, os, time from flask import Flask, request +from flask_restful import Api +from ResourceNetworkSlices import NetworkSliceService, NetworkSliceServices +from ResourceConnectionGroups import ConnectionGroup from YangValidator import YangValidator -DATA_FILE_PATH = os.environ.get('DATA_FILE_PATH') -if DATA_FILE_PATH is None: - raise Exception('DataFile({:s}) is empty'.format(str(DATA_FILE_PATH))) +LOG_LEVEL = logging.DEBUG -# TODO: if file not found, raise exception +logging.basicConfig( + level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s" +) +LOGGER = logging.getLogger(__name__) -with open(DATA_FILE_PATH, mode='r', encoding='UTF-8') as fp: - data = json.load(fp) +logging.getLogger('werkzeug').setLevel(logging.WARNING) + + + + +BASE_URL = '/restconf/data/etsi-qkd-sdn-node:qkd_node' yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) +def log_request(logger: logging.Logger, response): + timestamp = time.strftime("[%Y-%b-%d %H:%M]") + logger.info( + "%s %s %s %s %s", + timestamp, + request.remote_addr, + request.method, + request.full_path, + response.status, + ) + return response + app = Flask(__name__) +app.after_request(functools.partial(log_request, LOGGER)) + +api = Api(app, prefix=BASE_URL) +api.add_resource(NetworkSliceServices, "") +api.add_resource(NetworkSliceService, "/slice-service=") +api.add_resource( + ConnectionGroup, + "/slice-service=/connection-groups/connection-group=", +) + + + + + + + + + + def get_side_effect(url): steps = url.lstrip('https://').lstrip('http://').rstrip('/') -- GitLab From 6a4e9d2c3385f4d98e7781d99ce1fea6dc65c727 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 10 Jul 2025 13:47:42 +0000 Subject: [PATCH 31/82] CI/CD pipeline - QKD E2E test: - Minor code corrections --- src/tests/tools/mock_qkd_node/QkdNode.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/tools/mock_qkd_node/QkdNode.py b/src/tests/tools/mock_qkd_node/QkdNode.py index 7ab25876e..7c4f27696 100644 --- a/src/tests/tools/mock_qkd_node/QkdNode.py +++ b/src/tests/tools/mock_qkd_node/QkdNode.py @@ -36,11 +36,11 @@ class QkdNode(Resource): def post(self): json_request = request.get_json() - name = json_request["network-slice-services"]["slice-service"][0]["id"] - NETWORK_SLICES[name] = json_request + slice_id = json_request["network-slice-services"]["slice-service"][0]["id"] + QKD_NODE[slice_id] = json_request return make_response(jsonify({}), 201) - def delete(self, slice_id: str): - slice = NETWORK_SLICES.pop(slice_id, None) + def delete(self, slice_id : str): + slice = QKD_NODE.pop(slice_id, None) data, status = ({}, 404) if slice is None else (slice, 204) return make_response(jsonify(data), status) -- GitLab From 2ef6f8ebd50d92afd6af60d888f89a046ed642dd Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 10 Jul 2025 13:48:00 +0000 Subject: [PATCH 32/82] CI/CD pipeline - QKD E2E test: - Intermediate backup (cleaning code) --- src/tests/tools/mock_qkd_node/wsgi.py | 87 +++++++++++++++++---------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/src/tests/tools/mock_qkd_node/wsgi.py b/src/tests/tools/mock_qkd_node/wsgi.py index 184b9c006..a2574218a 100644 --- a/src/tests/tools/mock_qkd_node/wsgi.py +++ b/src/tests/tools/mock_qkd_node/wsgi.py @@ -124,33 +124,6 @@ def edit(from_dict, to_dict, create): else: to_dict[key] = value -def edit_side_effect(url, json, create): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") - - ip_port, _, _, header, *steps = parts - - module, root = header.split(':') - - assert(module == 'etsi-qkd-sdn-node') - assert(root == 'qkd_node') - - if not steps: - edit(json, nodes[ip_port]['node'], create) - return - - endpoint, *steps = steps - - if not steps: - edit(json[endpoint], nodes[ip_port][endpoint], create) - return - - raise Exception('Url too long') - @app.get('/', defaults={'path': ''}) @app.get("/") @app.get('/') @@ -167,14 +140,35 @@ def get(path): @app.post("/") @app.post('/') def post(path): - success = True - reason = '' try: - edit_side_effect(request.base_url, request.json, True) + steps = request.base_url.lstrip('https://').lstrip('http://').rstrip('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + return {'success': False, 'reason': f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}"} + + ip_port, _, _, header, *steps = parts + + module, root = header.split(':') + + assert module == 'etsi-qkd-sdn-node' + assert root == 'qkd_node' + + if not steps: + edit(request.json, nodes[ip_port]['node'], True) + return {'success': True, 'reason': ''} + + endpoint, *steps = steps + + if not steps: + edit(request.json[endpoint], nodes[ip_port][endpoint], True) + return {'success': True, 'reason': ''} + + return {'success': False, 'reason': 'Url too long'} + except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} + return {'success': False, 'reason': str(e)} @app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) @app.route("/", methods=['PUT', 'PATCH']) @@ -184,6 +178,33 @@ def patch(path): reason = '' try: edit_side_effect(request.base_url, request.json, False) +def edit_side_effect(url, json, create): + steps = url.lstrip('https://').lstrip('http://').rstrip('/') + parts = steps.split('/') + + # Ensure there are enough parts to unpack + if len(parts) < 4: + raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") + + ip_port, _, _, header, *steps = parts + + module, root = header.split(':') + + assert(module == 'etsi-qkd-sdn-node') + assert(root == 'qkd_node') + + if not steps: + edit(json, nodes[ip_port]['node'], create) + return + + endpoint, *steps = steps + + if not steps: + edit(json[endpoint], nodes[ip_port][endpoint], create) + return + + raise Exception('Url too long') + except Exception as e: reason = str(e) success = False -- GitLab From 621dc9955b31b3eb12b59e9d08fc6b116102a59b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 10 Jul 2025 15:13:37 +0000 Subject: [PATCH 33/82] QKD App: - Fixed error codes --- src/nbi/service/qkd_app/Resources.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/nbi/service/qkd_app/Resources.py b/src/nbi/service/qkd_app/Resources.py index fb4ec45d4..03a3d2fd2 100644 --- a/src/nbi/service/qkd_app/Resources.py +++ b/src/nbi/service/qkd_app/Resources.py @@ -12,16 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid -import json +import json, logging from flask import request +from flask.json import jsonify from flask_restful import Resource from common.proto.context_pb2 import Empty from common.proto.qkd_app_pb2 import App, QKDAppTypesEnum from common.Constants import DEFAULT_CONTEXT_NAME from context.client.ContextClient import ContextClient +from nbi.service._tools.HttpStatusCodes import HTTP_OK, HTTP_SERVERERROR from qkd_app.client.QKDAppClient import QKDAppClient +LOGGER = logging.getLogger(__name__) + class _Resource(Resource): def __init__(self) -> None: super().__init__() @@ -30,7 +33,7 @@ class _Resource(Resource): class Index(_Resource): def get(self): - return {'hello': 'world'} + return {} class ListDevices(_Resource): def get(self): @@ -81,18 +84,24 @@ class CreateQKDApp(_Resource): devices = self.context_client.ListDevices(Empty()).devices local_device = None + local_qkdn_id = app['local_qkdn_id'] + # This for-loop won't be necessary if Device ID is guaranteed to be the same as QKDN Id for device in devices: for config_rule in device.device_config.config_rules: if config_rule.custom.resource_key == '__node__': value = json.loads(config_rule.custom.resource_value) - qkdn_id = value['qkdn_id'] - if app['local_qkdn_id'] == qkdn_id: - local_device = device + if local_qkdn_id != value['qkdn_id']: continue + local_device = device break if local_device is None: - return {"status": "fail"} + MSG = 'Unable to find local_device for local_qkdn_id({:s})' + msg = MSG.format(str(local_qkdn_id)) + LOGGER.exception(msg) + response = jsonify({'error': msg}) + response.status_code = HTTP_SERVERERROR + return response external_app_src_dst = { 'app_id': {'context_id': {'context_uuid': {'uuid': DEFAULT_CONTEXT_NAME}}, 'app_uuid': {'uuid': ''}}, @@ -107,5 +116,6 @@ class CreateQKDApp(_Resource): self.qkd_app_client.RegisterApp(App(**external_app_src_dst)) - return {"status": "success"} - + response = jsonify({'status': 'success'}) + response.status_code = HTTP_OK + return response -- GitLab From 9623cb62dbb733e6f04fd7837b70ab262a30fd0d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 14 Jul 2025 16:51:36 +0000 Subject: [PATCH 34/82] CI/CD pipeline - QKD E2E test: - Added QKD node startup data - Updated TFS topology descriptor - Corrected TFS deploy specs - Progressed with CI/CD descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 100 ++++++++++++++---- src/tests/qkd_end2end/data/qkd-node-01.json | 36 +++++++ src/tests/qkd_end2end/data/qkd-node-02.json | 44 ++++++++ src/tests/qkd_end2end/data/qkd-node-03.json | 36 +++++++ .../qkd_end2end/{ => data}/topology_qkd.json | 12 +-- src/tests/qkd_end2end/deploy_specs.sh | 2 +- 6 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 src/tests/qkd_end2end/data/qkd-node-01.json create mode 100644 src/tests/qkd_end2end/data/qkd-node-02.json create mode 100644 src/tests/qkd_end2end/data/qkd-node-03.json rename src/tests/qkd_end2end/{ => data}/topology_qkd.json (89%) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index fd1c65b84..074f6b9ea 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -1,3 +1,17 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build, tag, and push the Docker image to the GitLab Docker registry build qkd_end2end: variables: @@ -26,30 +40,41 @@ end2end_test qkd_end2end: variables: TEST_NAME: 'qkd_end2end' stage: end2end_test - # Needs to run after build stage - needs: - - build qkd_end2end + # Disable to force running it after all other tasks + #needs: + # - build qkd_end2end before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f crdb nats mock_qkd context device pathcomp-frontend pathcomp-backend service qkd_end2end - - docker network rm teraflowbridge + - docker rm -f ${TEST_NAME} || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br script: - # Create Docker network for inter-container communication - - docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254 teraflowbridge - - # Pull necessary images - - docker pull "$CI_REGISTRY_IMAGE/context:latest" - - docker pull "$CI_REGISTRY_IMAGE/device:latest" - - docker pull "$CI_REGISTRY_IMAGE/service:latest" - - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:latest" - - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:latest" - - docker pull "$CI_REGISTRY_IMAGE/qkd_app:latest" - - docker pull "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" - - # Deploy CockroachDB (crdb) and NATS - - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node --insecure - - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + + # Deploy scenario with mock QKD Nodes + - docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + - > + docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 + --volume "$PWD/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" + mock-qkd-node:test + - > + docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 + --volume "$PWD/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" + mock-qkd-node:test + - > + docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 + --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" + mock-qkd-node:test + + # Check MicroK8s is ready + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + + + + # Wait for CockroachDB and NATS to initialize - echo "Waiting for CockroachDB to be ready..." @@ -91,9 +116,29 @@ end2end_test qkd_end2end: - sleep 10 # Run end-to-end tests for QKD application - - docker run --name ${TEST_NAME} -t --network=teraflowbridge --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: + # Dump configuration of the QKD Nodes (after_script) + - echo "[QKD-NODE-01] Config after_script:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config after_script:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config after_script:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # Dump logs of the QKD Nodes (after_script) + - docker logs qkd-node-01 + - docker logs qkd-node-02 + - docker logs qkd-node-03 + # Dump logs for TeraFlowSDN components - docker logs context - docker logs device @@ -101,15 +146,24 @@ end2end_test qkd_end2end: - docker logs pathcomp-backend - docker logs service - # Dump logs for QKD mock nodes - - docker logs mock_qkd + # Dump TeraFlowSDN component logs + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qkd-appservice -c server # Clean up - docker rm -f context device pathcomp-frontend pathcomp-backend service mock_qkd crdb nats - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force + # Clean old docker images + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' diff --git a/src/tests/qkd_end2end/data/qkd-node-01.json b/src/tests/qkd_end2end/data/qkd-node-01.json new file mode 100644 index 000000000..39d518faa --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-01.json @@ -0,0 +1,36 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000001-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "100", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "102", + "qkdi_att_point": { + "device": "QKD2", + "port": "201" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/data/qkd-node-02.json b/src/tests/qkd_end2end/data/qkd-node-02.json new file mode 100644 index 000000000..2ffa647b1 --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-02.json @@ -0,0 +1,44 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000002-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "200", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "201", + "qkdi_att_point": { + "device": "QKD1", + "port": "102" + }, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "203", + "qkdi_att_point": { + "device": "QKD3", + "port": "302" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/data/qkd-node-03.json b/src/tests/qkd_end2end/data/qkd-node-03.json new file mode 100644 index 000000000..276f2a2e0 --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-03.json @@ -0,0 +1,36 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000003-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "300", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "302", + "qkdi_att_point": { + "device": "QKD2", + "port": "203" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/topology_qkd.json b/src/tests/qkd_end2end/data/topology_qkd.json similarity index 89% rename from src/tests/qkd_end2end/topology_qkd.json rename to src/tests/qkd_end2end/data/topology_qkd.json index aa301baff..6c7b8c4ec 100644 --- a/src/tests/qkd_end2end/topology_qkd.json +++ b/src/tests/qkd_end2end/data/topology_qkd.json @@ -9,24 +9,24 @@ { "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], "device_config": {"config_rules": [ - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.101"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} ]} }, { "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], "device_config": {"config_rules": [ - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.102"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} ]} }, { "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], "device_config": {"config_rules": [ - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, - {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.103"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} ]} } diff --git a/src/tests/qkd_end2end/deploy_specs.sh b/src/tests/qkd_end2end/deploy_specs.sh index face2f5b0..9e06cd7b4 100755 --- a/src/tests/qkd_end2end/deploy_specs.sh +++ b/src/tests/qkd_end2end/deploy_specs.sh @@ -137,7 +137,7 @@ export CRDB_DEPLOY_MODE="single" export CRDB_DROP_DATABASE_IF_EXISTS="YES" # Disable flag for re-deploying CockroachDB from scratch. -export CRDB_REDEPLOY="YES" +export CRDB_REDEPLOY="" # ----- NATS ------------------------------------------------------------------- -- GitLab From 3b5b75855b6a52ec4251830cd8df5ae2b5dea695 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 14 Jul 2025 16:51:57 +0000 Subject: [PATCH 35/82] CI/CD pipeline - EUCNC24 E2E test: - Minor correction --- src/tests/eucnc24/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/eucnc24/.gitlab-ci.yml b/src/tests/eucnc24/.gitlab-ci.yml index 72ef8ff81..e63e0521b 100644 --- a/src/tests/eucnc24/.gitlab-ci.yml +++ b/src/tests/eucnc24/.gitlab-ci.yml @@ -171,7 +171,7 @@ end2end_test eucnc24: # Run end-to-end test: configure service IETF - > - docker run -t --rm --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-service-ietf-create.sh -- GitLab From b6a3e085329b06ca3601271d4f161d8f2bf6dac2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 14 Jul 2025 16:55:38 +0000 Subject: [PATCH 36/82] Test Tools - Mock QKD Node: - Initial functional version based on FreeCONF --- src/tests/tools/mock_qkd_node/Dockerfile | 41 +--- src/tests/tools/mock_qkd_node/QkdNode.py | 46 ---- .../tools/mock_qkd_node/YangValidator.py | 40 ---- src/tests/tools/mock_qkd_node/build.sh | 4 +- .../tools/mock_qkd_node/data/database-01.json | 20 -- .../tools/mock_qkd_node/data/database-02.json | 25 --- .../tools/mock_qkd_node/data/database-03.json | 20 -- src/tests/tools/mock_qkd_node/do_tests.sh | 74 ++++++ src/tests/tools/mock_qkd_node/qkd_node.py | 46 ++++ src/tests/tools/mock_qkd_node/requirements.in | 19 -- src/tests/tools/mock_qkd_node/run.sh | 31 ++- src/tests/tools/mock_qkd_node/startup.json | 16 ++ src/tests/tools/mock_qkd_node/tests.py | 43 ---- src/tests/tools/mock_qkd_node/tests.sh | 25 +++ src/tests/tools/mock_qkd_node/wsgi.py | 211 ------------------ 15 files changed, 201 insertions(+), 460 deletions(-) delete mode 100644 src/tests/tools/mock_qkd_node/QkdNode.py delete mode 100644 src/tests/tools/mock_qkd_node/YangValidator.py delete mode 100644 src/tests/tools/mock_qkd_node/data/database-01.json delete mode 100644 src/tests/tools/mock_qkd_node/data/database-02.json delete mode 100644 src/tests/tools/mock_qkd_node/data/database-03.json create mode 100755 src/tests/tools/mock_qkd_node/do_tests.sh create mode 100644 src/tests/tools/mock_qkd_node/qkd_node.py delete mode 100644 src/tests/tools/mock_qkd_node/requirements.in create mode 100644 src/tests/tools/mock_qkd_node/startup.json delete mode 100644 src/tests/tools/mock_qkd_node/tests.py create mode 100755 src/tests/tools/mock_qkd_node/tests.sh delete mode 100644 src/tests/tools/mock_qkd_node/wsgi.py diff --git a/src/tests/tools/mock_qkd_node/Dockerfile b/src/tests/tools/mock_qkd_node/Dockerfile index 510de0831..ef53aed75 100644 --- a/src/tests/tools/mock_qkd_node/Dockerfile +++ b/src/tests/tools/mock_qkd_node/Dockerfile @@ -14,45 +14,24 @@ FROM python:3.9-slim -# Install dependencies -RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install g++ git build-essential cmake libpcre2-dev python3-dev python3-cffi && \ - rm -rf /var/lib/apt/lists/* - -# Download, build and install libyang. Note that APT package is outdated -# - Ref: https://github.com/CESNET/libyang -# - Ref: https://github.com/CESNET/libyang-python/ -RUN mkdir -p /var/libyang -RUN git clone https://github.com/CESNET/libyang.git /var/libyang -WORKDIR /var/libyang -RUN git fetch -RUN git checkout v2.1.148 -RUN mkdir -p /var/libyang/build -WORKDIR /var/libyang/build -RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. -RUN make -RUN make install -RUN ldconfig - # Set Python to show logs as they occur ENV PYTHONUNBUFFERED=0 -# Get generic Python packages +# Get Python dependencies RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel -RUN python3 -m pip install --upgrade pip-tools +RUN python3 -m pip install https://github.com/freeconf/lang/releases/download/v0.1.0-alpha/freeconf-0.1.0-py3-none-any.whl +RUN fc-lang-install -v # Create component sub-folders, and copy content -RUN mkdir -p /var/teraflow/mock_qkd_node/{data,yang} -WORKDIR /var/teraflow/mock_qkd_node +RUN mkdir -p /var/mock_qkd_node/ +WORKDIR /var/mock_qkd_node COPY yang/. yang/ -COPY requirements.in requirements.in -COPY wsgi.py wsgi.py -COPY YangValidator.py YangValidator.py -# Get specific Python packages -RUN pip-compile --quiet --output-file=requirements.txt requirements.in -RUN python3 -m pip install -r requirements.txt +# Copy code and data +COPY ./yang ./yang +COPY ./startup.json ./startup.json +COPY ./qkd_node.py ./qkd_node.py # Start the service -ENTRYPOINT ["gunicorn", "--workers", "1", "--bind", "0.0.0.0:8080", "wsgi:app"] +ENTRYPOINT ["python", "qkd_node.py"] diff --git a/src/tests/tools/mock_qkd_node/QkdNode.py b/src/tests/tools/mock_qkd_node/QkdNode.py deleted file mode 100644 index 7c4f27696..000000000 --- a/src/tests/tools/mock_qkd_node/QkdNode.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# REST-API resource implementing minimal support for "IETF YANG Data Model for Transport Network Client Signals". -# Ref: https://www.ietf.org/archive/id/draft-ietf-ccamp-client-signal-yang-10.html - -import json, os -from flask import jsonify, make_response, request -from flask_restful import Resource - - -DATA_FILE_PATH = os.environ.get('DATA_FILE_PATH') -if DATA_FILE_PATH is None: - raise Exception('DataFile({:s}) is not defined'.format(str(DATA_FILE_PATH))) - -if not os.path.isfile(DATA_FILE_PATH): - raise Exception('DataFile({:s}) not found'.format(str(DATA_FILE_PATH))) - -with open(DATA_FILE_PATH, mode='r', encoding='UTF-8') as fp: - QKD_NODE = json.load(fp) - -class QkdNode(Resource): - def get(self): - return make_response(jsonify(QKD_NODE), 200) - - def post(self): - json_request = request.get_json() - slice_id = json_request["network-slice-services"]["slice-service"][0]["id"] - QKD_NODE[slice_id] = json_request - return make_response(jsonify({}), 201) - - def delete(self, slice_id : str): - slice = QKD_NODE.pop(slice_id, None) - data, status = ({}, 404) if slice is None else (slice, 204) - return make_response(jsonify(data), status) diff --git a/src/tests/tools/mock_qkd_node/YangValidator.py b/src/tests/tools/mock_qkd_node/YangValidator.py deleted file mode 100644 index 6214074c8..000000000 --- a/src/tests/tools/mock_qkd_node/YangValidator.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import libyang, os -from typing import Dict, Optional - -YANG_DIR = os.path.join(os.path.dirname(__file__), 'yang') - -class YangValidator: - def __init__(self, main_module : str, dependency_modules : [str]) -> None: - self._yang_context = libyang.Context(YANG_DIR) - - self._yang_module = self._yang_context.load_module(main_module) - mods = [self._yang_context.load_module(mod) for mod in dependency_modules] + [self._yang_module] - - for mod in mods: - mod.feature_enable_all() - - def parse_to_dict(self, message : Dict) -> Dict: - dnode : Optional[libyang.DNode] = self._yang_module.parse_data_dict( - message, validate_present=True, validate=True, strict=True - ) - if dnode is None: raise Exception('Unable to parse Message({:s})'.format(str(message))) - message = dnode.print_dict() - dnode.free() - return message - - def destroy(self) -> None: - self._yang_context.destroy() diff --git a/src/tests/tools/mock_qkd_node/build.sh b/src/tests/tools/mock_qkd_node/build.sh index 8360d93b9..d2b102209 100755 --- a/src/tests/tools/mock_qkd_node/build.sh +++ b/src/tests/tools/mock_qkd_node/build.sh @@ -17,5 +17,5 @@ cd $(dirname $0) docker build -t mock-qkd-node:test -f Dockerfile . -docker tag mock-qkd-node:test localhost:32000/tfs/mock-qkd-node:test -docker push localhost:32000/tfs/mock-qkd-node:test +#docker tag mock-qkd-node:test localhost:32000/tfs/mock-qkd-node:test +#docker push localhost:32000/tfs/mock-qkd-node:test diff --git a/src/tests/tools/mock_qkd_node/data/database-01.json b/src/tests/tools/mock_qkd_node/data/database-01.json deleted file mode 100644 index 1f78f7d6d..000000000 --- a/src/tests/tools/mock_qkd_node/data/database-01.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "node": {"qkdn_id": "00000001-0000-0000-0000-000000000000"}, - "qkdn_capabilities": {}, - "qkd_applications": {"qkd_app": []}, - "qkd_interfaces": { - "qkd_interface": [ - { - "qkdi_id": "100", - "qkdi_att_point": {}, - "qkdi_capabilities": {} - }, - { - "qkdi_id": "101", - "qkdi_att_point": {"device": current_ip, "port": "1001"}, - "qkdi_capabilities": {} - } - ] - }, - "qkd_links": {"qkd_link": []} -} diff --git a/src/tests/tools/mock_qkd_node/data/database-02.json b/src/tests/tools/mock_qkd_node/data/database-02.json deleted file mode 100644 index 4e3a00724..000000000 --- a/src/tests/tools/mock_qkd_node/data/database-02.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "node": {"qkdn_id": "00000002-0000-0000-0000-000000000000"}, - "qkdn_capabilities": {}, - "qkd_applications": {"qkd_app": []}, - "qkd_interfaces": { - "qkd_interface": [ - { - "qkdi_id": "200", - "qkdi_att_point": {}, - "qkdi_capabilities": {} - }, - { - "qkdi_id": "201", - "qkdi_att_point": {"device": current_ip, "port": "2001"}, - "qkdi_capabilities": {} - }, - { - "qkdi_id": "202", - "qkdi_att_point": {"device": current_ip, "port": "2002"}, - "qkdi_capabilities": {} - } - ] - }, - "qkd_links": {"qkd_link": []} -} diff --git a/src/tests/tools/mock_qkd_node/data/database-03.json b/src/tests/tools/mock_qkd_node/data/database-03.json deleted file mode 100644 index b6dbb7aa5..000000000 --- a/src/tests/tools/mock_qkd_node/data/database-03.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "node": {"qkdn_id": "00000003-0000-0000-0000-000000000000"}, - "qkdn_capabilities": {}, - "qkd_applications": {"qkd_app": []}, - "qkd_interfaces": { - "qkd_interface": [ - { - "qkdi_id": "300", - "qkdi_att_point": {}, - "qkdi_capabilities": {} - }, - { - "qkdi_id": "301", - "qkdi_att_point": {"device": current_ip, "port": "3001"}, - "qkdi_capabilities": {} - } - ] - }, - "qkd_links": {"qkd_link": []} -} diff --git a/src/tests/tools/mock_qkd_node/do_tests.sh b/src/tests/tools/mock_qkd_node/do_tests.sh new file mode 100755 index 000000000..9ace3467b --- /dev/null +++ b/src/tests/tools/mock_qkd_node/do_tests.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Build the new Docker image +echo "Building..." +docker build -t mock-qkd-node:test -f Dockerfile . +echo + +# Run container +echo "Running..." +docker run -d --name qkd-node-01 -p 7777:7777 mock-qkd-node:test +echo + +# Give 5 seconds for container to start +echo "Waiting 3 seconds..." +for i in {1..3}; do + printf "%c" "." + sleep 1 +done +echo + +# Dumping containers before... +echo "Dumping containers before..." +docker ps -a +echo + +# Dumping logs before... +echo "Dumping logs before..." +docker logs qkd-node-01 +echo + +# Read data from startup +echo "Reading before..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Update data +echo "Updating 1..." +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo + +# Read data after 1 update +echo "Reading after 1..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Update data 2 +echo "Updating 2..." +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://localhost:7777/restconf/data/etsi-qkd-sdn-node:qkd_node +echo + +# Read data after 2 update +echo "Reading after 2..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Dumping containers after... +echo "Dumping containers after..." +docker ps -a +echo + +# Dumping logs after... +echo "Dumping logs after..." +docker logs qkd-node-01 +echo + +# Destroy container +echo "Destroying..." +docker rm --force qkd-node-01 +echo + +echo "Bye!!" diff --git a/src/tests/tools/mock_qkd_node/qkd_node.py b/src/tests/tools/mock_qkd_node/qkd_node.py new file mode 100644 index 000000000..9834a259b --- /dev/null +++ b/src/tests/tools/mock_qkd_node/qkd_node.py @@ -0,0 +1,46 @@ +from freeconf import restconf, source, device, parser, node, source, nodeutil +from threading import Event + +YANG_PATH = './yang' +YANG_MDL = 'etsi-qkd-sdn-node' +STARTUP_FILE = './startup.json' + +def main(): + # specify all the places where you store YANG files + yang_path = source.any( + source.path(YANG_PATH), # director to your local *.yang files + source.restconf_internal_ypath() # required for restconf protocol support + ) + + # load and validate your YANG file(s) + yang_module = parser.load_module_file(yang_path, YANG_MDL) + + # device hosts one or more management "modules" into a single instance that you + # want to export in the management interface + _device = device.Device(yang_path) + + # connect your application to your management implementation. + # there are endless ways to to build your management interface from code generation, + # to reflection and any combination there of. A lot more information in docs. + handler = nodeutil.Node(dict()) + + # connect parsed YANG to your management implementation. Browser is a powerful way + # to dynamically control your application can can be useful in unit tests or other contexts + # but here we construct it to serve our management API + browser = node.Browser(yang_module, handler) + + # register your app management browser in device. Device can hold any number of browsers + _device.add_browser(browser) + + # select RESTCONF as management protocol. gNMI is option as well or any custom or + # future protocols + restconf.Server(_device) + + # this will apply configuration including starting the RESTCONF web server + _device.apply_startup_config_file(STARTUP_FILE) + + # simple python trick to wait until ctrl-c shutdown + Event().wait() + +if __name__ == '__main__': + main() diff --git a/src/tests/tools/mock_qkd_node/requirements.in b/src/tests/tools/mock_qkd_node/requirements.in deleted file mode 100644 index cb2f89992..000000000 --- a/src/tests/tools/mock_qkd_node/requirements.in +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flask==2.1.3 -Flask-HTTPAuth==4.5.0 -Flask-RESTful==0.3.9 -gunicorn==23.0.0 -werkzeug==2.3.7 diff --git a/src/tests/tools/mock_qkd_node/run.sh b/src/tests/tools/mock_qkd_node/run.sh index ecef1d5d5..0690696f2 100755 --- a/src/tests/tools/mock_qkd_node/run.sh +++ b/src/tests/tools/mock_qkd_node/run.sh @@ -13,7 +13,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Make folder containing the script the root folder for its execution -cd $(dirname $0) +# Cleanup +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br -python MockIetfActnSdnCtrl.py +# Create Docker network +docker network create \ + --driver bridge \ + --subnet=172.254.250.0/24 \ + --gateway=172.254.250.254 \ + --ip-range=172.254.250.0/24 \ + qkd-node-br + +#--volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_t1.xml:/confd/examples.confd/OC23/platform.xml" + +# Create QKD Nodes +docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ + --volume "$PWD/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ + --volume "$PWD/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ + --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test + +# Dump QKD Node Docker containers +docker ps -a + +echo "Bye!" diff --git a/src/tests/tools/mock_qkd_node/startup.json b/src/tests/tools/mock_qkd_node/startup.json new file mode 100644 index 000000000..f1670f1ba --- /dev/null +++ b/src/tests/tools/mock_qkd_node/startup.json @@ -0,0 +1,16 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000000-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": {}, + "qkd_interfaces": {}, + "qkd_links": {} + } + } +} diff --git a/src/tests/tools/mock_qkd_node/tests.py b/src/tests/tools/mock_qkd_node/tests.py deleted file mode 100644 index b112595dd..000000000 --- a/src/tests/tools/mock_qkd_node/tests.py +++ /dev/null @@ -1,43 +0,0 @@ -import json -from mock import requests -import pyangbind.lib.pybindJSON as enc -from pyangbind.lib.serialise import pybindJSONDecoder as dec -from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node - -module = etsi_qkd_sdn_node() -url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:qkd_node/' - -# Get node all info -z = requests.get(url).json() -var = dec.load_json(z, None, None, obj=module) -print(enc.dumps(var)) - -# Reset module variable because it is already filled -module = etsi_qkd_sdn_node() - -# Get node basic info -node = module.qkd_node -z = requests.get(url).json() -var = dec.load_json(z, None, None, obj=node) -print(enc.dumps(var)) - -# Get all apps -apps = node.qkd_applications -z = requests.get(url + 'qkd_applications').json() -var = dec.load_json(z, None, None, obj=apps) -print(enc.dumps(var)) - -# Edit app 0 -app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] -app.client_app_id = 'id_0' -requests.put(url + 'qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) - -# Create app 1 -app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') -requests.post(url + 'qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) - -# Get all apps -apps = node.qkd_applications -z = requests.get(url + 'qkd_applications').json() -var = dec.load_json(z, None, None, obj=apps) -print(enc.dumps(var)) diff --git a/src/tests/tools/mock_qkd_node/tests.sh b/src/tests/tools/mock_qkd_node/tests.sh new file mode 100755 index 000000000..ea19eaa91 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +echo "[QKD-NODE-01] Reading data on startup..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + +echo "[QKD-NODE-01] Updating location (from root)..." +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo + +echo "[QKD-NODE-01] Reading after update 1..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + +echo "[QKD-NODE-01] Updating location (from path)..." +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:qkd_node +echo + +echo "[QKD-NODE-01] Reading final value..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + diff --git a/src/tests/tools/mock_qkd_node/wsgi.py b/src/tests/tools/mock_qkd_node/wsgi.py deleted file mode 100644 index a2574218a..000000000 --- a/src/tests/tools/mock_qkd_node/wsgi.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools, json, logging, os, time -from flask import Flask, request -from flask_restful import Api -from ResourceNetworkSlices import NetworkSliceService, NetworkSliceServices -from ResourceConnectionGroups import ConnectionGroup -from YangValidator import YangValidator - -LOG_LEVEL = logging.DEBUG - -logging.basicConfig( - level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s" -) -LOGGER = logging.getLogger(__name__) - -logging.getLogger('werkzeug').setLevel(logging.WARNING) - - - - -BASE_URL = '/restconf/data/etsi-qkd-sdn-node:qkd_node' - -yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - -def log_request(logger: logging.Logger, response): - timestamp = time.strftime("[%Y-%b-%d %H:%M]") - logger.info( - "%s %s %s %s %s", - timestamp, - request.remote_addr, - request.method, - request.full_path, - response.status, - ) - return response - -app = Flask(__name__) -app.after_request(functools.partial(log_request, LOGGER)) - -api = Api(app, prefix=BASE_URL) -api.add_resource(NetworkSliceServices, "") -api.add_resource(NetworkSliceService, "/slice-service=") -api.add_resource( - ConnectionGroup, - "/slice-service=/connection-groups/connection-group=", -) - - - - - - - - - - - -def get_side_effect(url): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") - - ip_port, _, _, header, *steps = parts - - header_splitted = header.split(':') - - module = header_splitted[0] - assert(module == 'etsi-qkd-sdn-node') - - if ip_port.startswith('127.0.0.1'): - ip_port = ip_port.replace('127.0.0.1', current_ip) - - tree = {'qkd_node': nodes[ip_port]['node'].copy()} - - if len(header_splitted) == 1 or not header_splitted[1]: - value = nodes[ip_port].copy() - value.pop('node') - tree['qkd_node'].update(value) - - return tree, tree - - root = header_splitted[1] - assert(root == 'qkd_node') - - if not steps: - return tree, tree - - endpoint, *steps = steps - - value = nodes[ip_port][endpoint] - - if not steps: - return_value = {endpoint: value} - tree['qkd_node'].update(return_value) - - return return_value, tree - - raise Exception('Url too long') - -def edit(from_dict, to_dict, create): - for key, value in from_dict.items(): - if isinstance(value, dict): - if key not in to_dict and create: - to_dict[key] = {} - edit(from_dict[key], to_dict[key], create) - elif isinstance(value, list): - to_dict[key].extend(value) - else: - to_dict[key] = value - -@app.get('/', defaults={'path': ''}) -@app.get("/") -@app.get('/') -def get(path): - try: - msg, msg_validate = get_side_effect(request.base_url) - print(msg_validate) - yang_validator.parse_to_dict(msg_validate) - return msg - except ValueError as e: - return {'error': str(e)}, 400 - -@app.post('/', defaults={'path': ''}) -@app.post("/") -@app.post('/') -def post(path): - try: - steps = request.base_url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - return {'success': False, 'reason': f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}"} - - ip_port, _, _, header, *steps = parts - - module, root = header.split(':') - - assert module == 'etsi-qkd-sdn-node' - assert root == 'qkd_node' - - if not steps: - edit(request.json, nodes[ip_port]['node'], True) - return {'success': True, 'reason': ''} - - endpoint, *steps = steps - - if not steps: - edit(request.json[endpoint], nodes[ip_port][endpoint], True) - return {'success': True, 'reason': ''} - - return {'success': False, 'reason': 'Url too long'} - - except Exception as e: - return {'success': False, 'reason': str(e)} - -@app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) -@app.route("/", methods=['PUT', 'PATCH']) -@app.route('/', methods=['PUT', 'PATCH']) -def patch(path): - success = True - reason = '' - try: - edit_side_effect(request.base_url, request.json, False) -def edit_side_effect(url, json, create): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - parts = steps.split('/') - - # Ensure there are enough parts to unpack - if len(parts) < 4: - raise ValueError(f"Expected at least 4 parts in the URL, got {len(parts)}: {steps}") - - ip_port, _, _, header, *steps = parts - - module, root = header.split(':') - - assert(module == 'etsi-qkd-sdn-node') - assert(root == 'qkd_node') - - if not steps: - edit(json, nodes[ip_port]['node'], create) - return - - endpoint, *steps = steps - - if not steps: - edit(json[endpoint], nodes[ip_port][endpoint], create) - return - - raise Exception('Url too long') - - except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} -- GitLab From 209be8f5a0076fc97c0e37d958d255a43f68c722 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 11:16:10 +0000 Subject: [PATCH 37/82] Deployment scripts: - Clean-up old Zookeeper - Align deployment to CockroachDB - Update/Cleanup settings --- deploy/all.sh | 20 ++++ deploy/crdb.sh | 2 +- deploy/kafka.sh | 140 +++++++++++++++---------- deploy/tfs.sh | 30 +----- manifests/cockroachdb/single-node.yaml | 2 +- manifests/kafka/01-zookeeper.yaml | 53 ---------- manifests/kafka/02-kafka.yaml | 60 ----------- manifests/kafka/single-node.yaml | 97 +++++++++++++++++ my_deploy.sh | 30 +++--- src/common/tools/kafka/Variables.py | 2 +- 10 files changed, 226 insertions(+), 210 deletions(-) delete mode 100644 manifests/kafka/01-zookeeper.yaml delete mode 100644 manifests/kafka/02-kafka.yaml create mode 100644 manifests/kafka/single-node.yaml diff --git a/deploy/all.sh b/deploy/all.sh index 93018d3ce..a284287bc 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -151,6 +151,26 @@ export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} export NATS_REDEPLOY=${NATS_REDEPLOY:-""} +# ----- Apache Kafka ----------------------------------------------------------- + +# If not already set, set the namespace where Kafka will be deployed. +export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} + +# If not already set, set the external port Kafka Client interface will be exposed to. +export KFK_EXT_PORT_CLIENT=${KFK_EXT_PORT_CLIENT:-"9092"} + +# If not already set, set Kafka installation mode. Accepted values are: 'single'. +# - If KFK_DEPLOY_MODE is "single", Kafka is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# NOTE: Production mode is still not supported. Will be provided in the future. +export KFK_DEPLOY_MODE=${KFK_DEPLOY_MODE:-"single"} + +# If not already set, disable flag for re-deploying Kafka from scratch. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! +# If KFK_REDEPLOY is "YES", the message broker will be dropped while checking/deploying Kafka. +export KFK_REDEPLOY=${KFK_REDEPLOY:-""} + + # ----- QuestDB ---------------------------------------------------------------- # If not already set, set the namespace where QuestDB will be deployed. diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 6866f484f..0cff8594d 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -66,7 +66,7 @@ CRDB_MANIFESTS_PATH="manifests/cockroachdb" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${CRDB_NAMESPACE}/manifests" -mkdir -p $TMP_MANIFESTS_FOLDER +mkdir -p ${TMP_MANIFESTS_FOLDER} function crdb_deploy_single() { echo "CockroachDB Namespace" diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 5dee8afae..77300c216 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -13,17 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. + ######################################################################################################################## # Read deployment settings ######################################################################################################################## -# If not already set, set the namespace where Apache Kafka will be deployed. +# If not already set, set the namespace where Kafka will be deployed. export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} -# If not already set, set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} +# If not already set, set the external port Kafka client interface will be exposed to. +export KFK_EXT_PORT_CLIENT=${KFK_EXT_PORT_CLIENT:-"9092"} + +# If not already set, set Kafka installation mode. Accepted values are: 'single'. +# - If KFK_DEPLOY_MODE is "single", Kafka is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# NOTE: Production mode is still not supported. Will be provided in the future. +export KFK_DEPLOY_MODE=${KFK_DEPLOY_MODE:-"single"} -# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost. +# If not already set, disable flag for re-deploying Kafka from scratch. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! +# If KFK_REDEPLOY is "YES", the message broker will be dropped while checking/deploying Kafka. export KFK_REDEPLOY=${KFK_REDEPLOY:-""} @@ -31,61 +40,86 @@ export KFK_REDEPLOY=${KFK_REDEPLOY:-""} # Automated steps start here ######################################################################################################################## - # Constants - TMP_FOLDER="./tmp" - KFK_MANIFESTS_PATH="manifests/kafka" - KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" - KFK_MANIFEST="02-kafka.yaml" - - # Create a tmp folder for files modified during the deployment - TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" - mkdir -p ${TMP_MANIFESTS_FOLDER} +# Constants +TMP_FOLDER="./tmp" +KFK_MANIFESTS_PATH="manifests/kafka" -function kafka_deploy() { - # copy zookeeper and kafka manifest files to temporary manifest location - cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" - cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" +mkdir -p ${TMP_MANIFESTS_FOLDER} - # echo "Apache Kafka Namespace" - echo "Delete Apache Kafka Namespace" - kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found - - echo "Create Apache Kafka Namespace" +function kfk_deploy_single() { + echo "Kafka Namespace" + echo ">>> Create Kafka Namespace (if missing)" kubectl create namespace ${KFK_NAMESPACE} + echo + + echo "Kafka (single-mode)" + echo ">>> Checking if Kafka is deployed..." + if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/kafka &> /dev/null; then + echo ">>> Kafka is present; skipping step." + else + echo ">>> Deploy Kafka" + cp "${KFK_MANIFESTS_PATH}/single-node.yaml" "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + #sed -i "s//${KFK_NAMESPACE}/" "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + + echo ">>> Waiting Kakfa statefulset to be created..." + while ! kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; do + printf "%c" "." + sleep 1 + done + + # Wait for statefulset condition "Available=True" does not work + # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error: + # "error: readyReplicas is not found" + # Workaround: Check the pods are ready + #echo ">>> Kafka statefulset created. Waiting for readiness condition..." + #kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/kafka + #kubectl wait --namespace ${KGK_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \ + # statefulset/kafka + echo ">>> Kakfa statefulset created. Waiting Kakfa pods to be created..." + while ! kubectl get --namespace ${KFK_NAMESPACE} pod/kafka-0 &> /dev/null; do + printf "%c" "." + sleep 1 + done + kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Ready --timeout=300s pod/kafka-0 + + # Wait for Kafka to notify "Kafka Server started" + echo ">>> Kakfa pods created. Waiting Kakfa Server be started..." + while ! kubectl --namespace $KFK_NAMESPACE logs pod/kafka-0 -c kafka 2>&1 | grep -q 'Kafka Server started'; do + printf "%c" "." + sleep 1 + done + fi + echo +} - # echo ">>> Deplying Apache Kafka Zookeeper" - # Kafka zookeeper service should be deployed before the kafka service - kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" - - #KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically - #KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') - - # Kafka service should be deployed after the zookeeper service - #sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - sed -i "s//${KFK_NAMESPACE}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - - # echo ">>> Deploying Apache Kafka Broker" - kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - # echo ">>> Verifing Apache Kafka deployment" - sleep 5 - # KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) - # if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then - # echo "Deployment Error: \n $KFK_PODS_STATUS" - # else - # echo "$KFK_PODS_STATUS" - # fi +function kfk_undeploy_single() { + echo "Kafka (single-mode)" + echo ">>> Checking if Kafka is deployed..." + if kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; then + echo ">>> Undeploy Kafka" + kubectl delete --namespace ${KFK_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" --ignore-not-found + else + echo ">>> Kafka is not present; skipping step." + fi + echo + + echo "Kafka Namespace" + echo ">>> Delete Kafka Namespace (if exists)" + echo "NOTE: this step might take few minutes to complete!" + kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found + echo } -echo ">>> Apache Kafka" -echo "Checking if Apache Kafka is deployed ... " -if [ "$KFK_REDEPLOY" == "YES" ]; then - echo "Redeploying kafka namespace" - kafka_deploy -elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then - echo "Apache Kafka already present; skipping step." +if [ "$KFK_DEPLOY_MODE" == "single" ]; then + if [ "$KFK_REDEPLOY" == "YES" ]; then + kfk_undeploy_single + fi + + kfk_deploy_single else - echo "Kafka namespace doesn't exists. Deploying kafka namespace" - kafka_deploy + echo "Unsupported value: KFK_DEPLOY_MODE=$KFK_DEPLOY_MODE" fi -echo diff --git a/deploy/tfs.sh b/deploy/tfs.sh index b73bbbf81..917edb6fa 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -51,12 +51,6 @@ export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""} # If not already set, set the namespace where CockroackDB will be deployed. export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"} -# If not already set, set the external port CockroackDB Postgre SQL interface will be exposed to. -export CRDB_EXT_PORT_SQL=${CRDB_EXT_PORT_SQL:-"26257"} - -# If not already set, set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. -export CRDB_EXT_PORT_HTTP=${CRDB_EXT_PORT_HTTP:-"8081"} - # If not already set, set the database username to be used by Context. export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"} @@ -69,27 +63,12 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} # If not already set, set the namespace where NATS will be deployed. export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"} -# If not already set, set the external port NATS Client interface will be exposed to. -export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} - -# If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. -export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} - # ----- QuestDB ---------------------------------------------------------------- # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} -# If not already set, set the external port QuestDB Postgre SQL interface will be exposed to. -export QDB_EXT_PORT_SQL=${QDB_EXT_PORT_SQL:-"8812"} - -# If not already set, set the external port QuestDB Influx Line Protocol interface will be exposed to. -export QDB_EXT_PORT_ILP=${QDB_EXT_PORT_ILP:-"9009"} - -# If not already set, set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. -export QDB_EXT_PORT_HTTP=${QDB_EXT_PORT_HTTP:-"9000"} - # If not already set, set the database username to be used for QuestDB. export QDB_USERNAME=${QDB_USERNAME:-"admin"} @@ -114,14 +93,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} # ----- Apache Kafka ------------------------------------------------------ -# If not already set, set the namespace where Apache Kafka will be deployed. +# If not already set, set the namespace where Kafka will be deployed. export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} -# If not already set, set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} - -# If not already set, if flag is YES, Apache Kafka will be redeployed and topic will be lost. -export KFK_REDEPLOY=${KFK_REDEPLOY:-""} ######################################################################################################################## # Automated steps start here @@ -154,7 +128,7 @@ kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type= printf "\n" echo ">>> Create Secret with Apache Kafka..." -KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') +KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-public -o 'jsonpath={.spec.ports[0].port}') kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ --from-literal=KFK_SERVER_PORT=${KFK_SERVER_PORT} diff --git a/manifests/cockroachdb/single-node.yaml b/manifests/cockroachdb/single-node.yaml index fec9b5289..ed297d77c 100644 --- a/manifests/cockroachdb/single-node.yaml +++ b/manifests/cockroachdb/single-node.yaml @@ -61,7 +61,7 @@ spec: containers: - name: cockroachdb image: cockroachdb/cockroach:latest-v22.2 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent args: - start-single-node ports: diff --git a/manifests/kafka/01-zookeeper.yaml b/manifests/kafka/01-zookeeper.yaml deleted file mode 100644 index f2cfb4f38..000000000 --- a/manifests/kafka/01-zookeeper.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - labels: - app: zookeeper-service - name: zookeeper-service -spec: - type: ClusterIP - ports: - - name: zookeeper-port - port: 2181 - #nodePort: 30181 - #targetPort: 2181 - selector: - app: zookeeper ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: zookeeper - name: zookeeper -spec: - replicas: 1 - selector: - matchLabels: - app: zookeeper - template: - metadata: - labels: - app: zookeeper - spec: - containers: - - image: wurstmeister/zookeeper - imagePullPolicy: IfNotPresent - name: zookeeper - ports: - - containerPort: 2181 diff --git a/manifests/kafka/02-kafka.yaml b/manifests/kafka/02-kafka.yaml deleted file mode 100644 index 066f0151a..000000000 --- a/manifests/kafka/02-kafka.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - labels: - app: kafka-broker - name: kafka-service -spec: - ports: - - port: 9092 - selector: - app: kafka-broker ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: kafka-broker - name: kafka-broker -spec: - replicas: 1 - selector: - matchLabels: - app: kafka-broker - template: - metadata: - labels: - app: kafka-broker - spec: - hostname: kafka-broker - containers: - - env: - - name: KAFKA_BROKER_ID - value: "1" - - name: KAFKA_ZOOKEEPER_CONNECT - #value: :2181 - value: zookeeper-service..svc.cluster.local:2181 - - name: KAFKA_LISTENERS - value: PLAINTEXT://:9092 - - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka-service..svc.cluster.local:9092 - image: wurstmeister/kafka - imagePullPolicy: IfNotPresent - name: kafka-broker - ports: - - containerPort: 9092 diff --git a/manifests/kafka/single-node.yaml b/manifests/kafka/single-node.yaml new file mode 100644 index 000000000..4c435c11b --- /dev/null +++ b/manifests/kafka/single-node.yaml @@ -0,0 +1,97 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: kafka-public + labels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka +spec: + type: ClusterIP + selector: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + ports: + - name: clients + port: 9092 + protocol: TCP + targetPort: 9092 + - name: control-plane + port: 9093 + protocol: TCP + targetPort: 9093 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka +spec: + selector: + matchLabels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + serviceName: "kafka-public" + replicas: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + spec: + terminationGracePeriodSeconds: 10 + restartPolicy: Always + containers: + - name: kafka + image: bitnami/kafka:latest + imagePullPolicy: IfNotPresent + ports: + - name: clients + containerPort: 9092 + - name: control-plane + containerPort: 9093 + env: + #- name: KAFKA_BROKER_ID + # value: "1" + #- name: KAFKA_ZOOKEEPER_CONNECT + # value: zookeeper-service..svc.cluster.local:2181 + #- name: KAFKA_LISTENERS + # value: PLAINTEXT://:9092 + #- name: KAFKA_ADVERTISED_LISTENERS + # value: PLAINTEXT://kafka-service..svc.cluster.local:9092 + - name: KAFKA_CFG_NODE_ID + value: "1" + - name: KAFKA_CFG_PROCESS_ROLES + value: "controller,broker" + - name: KAFKA_CFG_LISTENERS + value: "PLAINTEXT://:9092,CONTROLLER://:9093" + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT" + - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES + value: "CONTROLLER" + - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS + value: "1@kafka-0:9093" + resources: + requests: + cpu: "250m" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/my_deploy.sh b/my_deploy.sh index 969caa94f..d3384276d 100644 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -137,7 +137,7 @@ export CRDB_DEPLOY_MODE="single" export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. -export CRDB_REDEPLOY="YES" +export CRDB_REDEPLOY="" # ----- NATS ------------------------------------------------------------------- @@ -159,6 +159,22 @@ export NATS_DEPLOY_MODE="single" export NATS_REDEPLOY="" +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + +# Set the port Apache Kafka server will be exposed to. +export KFK_EXT_PORT_CLIENT="9092" + +# Set Kafka installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/kafka.sh for additional details +export KFK_DEPLOY_MODE="single" + +# Disable flag for re-deploying Kafka from scratch. +export KFK_REDEPLOY="" + + # ----- QuestDB ---------------------------------------------------------------- # Set the namespace where QuestDB will be deployed. @@ -199,15 +215,3 @@ export PROM_EXT_PORT_HTTP="9090" # Set the external port Grafana HTTP Dashboards will be exposed to. export GRAF_EXT_PORT_HTTP="3000" - - -# ----- Apache Kafka ----------------------------------------------------------- - -# Set the namespace where Apache Kafka will be deployed. -export KFK_NAMESPACE="kafka" - -# Set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT="9092" - -# Set the flag to YES for redeploying of Apache Kafka -export KFK_REDEPLOY="" diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index eac8dfc27..7bb131dd6 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -20,7 +20,7 @@ from common.Settings import get_setting LOGGER = logging.getLogger(__name__) -KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' +KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-public.{:s}.svc.cluster.local:{:s}' KAFKA_TOPIC_NUM_PARTITIONS = 1 KAFKA_TOPIC_REPLICATION_FACTOR = 1 -- GitLab From 86b49b7e49a62f8e470d8e0a74c9878ccce4bdd2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 11:16:58 +0000 Subject: [PATCH 38/82] QKD E2E Test: - Added deploy-qkd-nodes.sh --- src/tests/qkd_end2end/deploy-qkd-nodes.sh | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 src/tests/qkd_end2end/deploy-qkd-nodes.sh diff --git a/src/tests/qkd_end2end/deploy-qkd-nodes.sh b/src/tests/qkd_end2end/deploy-qkd-nodes.sh new file mode 100755 index 000000000..d4dc25752 --- /dev/null +++ b/src/tests/qkd_end2end/deploy-qkd-nodes.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br + +# Create Docker network +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + +# Create QKD Nodes +docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test + +# Dump QKD Node Docker containers +docker ps -a + +echo "Bye!" -- GitLab From 86ce70f5012a411e1aa8d7f7cf8787eb6082dfd5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 11:17:25 +0000 Subject: [PATCH 39/82] Mock QKD Node: - Converted run.sh to start a single node for testing purposes --- src/tests/tools/mock_qkd_node/run.sh | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/tests/tools/mock_qkd_node/run.sh b/src/tests/tools/mock_qkd_node/run.sh index 0690696f2..900f0994f 100755 --- a/src/tests/tools/mock_qkd_node/run.sh +++ b/src/tests/tools/mock_qkd_node/run.sh @@ -14,29 +14,14 @@ # limitations under the License. # Cleanup -docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker rm --force qkd-node docker network rm --force qkd-node-br # Create Docker network -docker network create \ - --driver bridge \ - --subnet=172.254.250.0/24 \ - --gateway=172.254.250.254 \ - --ip-range=172.254.250.0/24 \ - qkd-node-br +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br -#--volume "$PWD/src/tests/${TEST_NAME}/node-agents-config/platform_t1.xml:/confd/examples.confd/OC23/platform.xml" - -# Create QKD Nodes -docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ - --volume "$PWD/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ - mock-qkd-node:test -docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ - --volume "$PWD/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ - mock-qkd-node:test -docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ - --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ - mock-qkd-node:test +# Create QKD Node +docker run --detach --name qkd-node --network qkd-node-br --ip 172.254.250.101 mock-qkd-node:test # Dump QKD Node Docker containers docker ps -a -- GitLab From 0682743603ce8064b6e3e7ecaad18c38edac1ca5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 11:26:19 +0000 Subject: [PATCH 40/82] QKD E2E Test: - Corrected topology descriptor --- src/tests/qkd_end2end/data/topology_qkd.json | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/tests/qkd_end2end/data/topology_qkd.json b/src/tests/qkd_end2end/data/topology_qkd.json index 6c7b8c4ec..fe8aa367c 100644 --- a/src/tests/qkd_end2end/data/topology_qkd.json +++ b/src/tests/qkd_end2end/data/topology_qkd.json @@ -32,21 +32,21 @@ } ], "links": [ - {"link_id": {"link_uuid": {"uuid": "QKD1/10.0.2.10:1001==QKD2/10.0.2.10:2001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}} + {"link_id": {"link_uuid": {"uuid": "QKD1/QKD2:201==QKD2/QKD1:102"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}} ]}, - {"link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2001==QKD1/10.0.2.10:1001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}} + {"link_id": {"link_uuid": {"uuid": "QKD2/QKD1:102==QKD1/QKD2:201"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}} ]}, - {"link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2002==QKD3/10.0.2.10:3001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}}, - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}} + {"link_id": {"link_uuid": {"uuid": "QKD2/QKD3:302==QKD3/QKD2:203"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} ]}, - {"link_id": {"link_uuid": {"uuid": "QKD3/10.0.2.10:3001==QKD2/10.0.2.10:2002"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}} + {"link_id": {"link_uuid": {"uuid": "QKD3/QKD2:203==QKD2/QKD3:302"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}} ]} ] } -- GitLab From 33a983c2b91e37424a76f027a943367aa3db768f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 15:04:56 +0000 Subject: [PATCH 41/82] QKD E2E Test: - Added descriptors for direct and virtual QKD links - Added commands.txt with useful commands - Added redeploy scripts --- src/tests/qkd_end2end/commands.txt | 8 ++++ .../data/direct-link-qkd1-qkd2.json | 16 ++++++++ .../data/direct-link-qkd2-qkd3.json | 16 ++++++++ .../data/virtual-link-qkd1-qkd3.json | 16 ++++++++ src/tests/qkd_end2end/dump_logs.sh | 9 +++++ src/tests/qkd_end2end/redeploy-all.sh | 40 +++++++++++++++++++ ...loy-qkd-nodes.sh => redeploy-qkd-nodes.sh} | 0 7 files changed, 105 insertions(+) create mode 100644 src/tests/qkd_end2end/commands.txt create mode 100644 src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json create mode 100644 src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json create mode 100644 src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json create mode 100755 src/tests/qkd_end2end/dump_logs.sh create mode 100755 src/tests/qkd_end2end/redeploy-all.sh rename src/tests/qkd_end2end/{deploy-qkd-nodes.sh => redeploy-qkd-nodes.sh} (100%) diff --git a/src/tests/qkd_end2end/commands.txt b/src/tests/qkd_end2end/commands.txt new file mode 100644 index 000000000..36fb03cdf --- /dev/null +++ b/src/tests/qkd_end2end/commands.txt @@ -0,0 +1,8 @@ + +src/tests/qkd_end2end/redeploy-all.sh +src/tests/qkd_end2end/redeploy-tfs.sh +src/tests/qkd_end2end/redeploy-qkd-nodes.sh + +python src/qkd_app/tests/test_create_apps.py + +src/tests/qkd_end2end/dump_logs.sh diff --git a/src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json b/src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json new file mode 100644 index 000000000..873ce6547 --- /dev/null +++ b/src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "direct-link-QKD1-QKD2"} + }, + "name": "direct-link-QKD1-QKD2", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json b/src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json new file mode 100644 index 000000000..f9570ce8d --- /dev/null +++ b/src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "direct-link-QKD2-QKD3"} + }, + "name": "direct-link-QKD2-QKD3", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json b/src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json new file mode 100644 index 000000000..206a8a279 --- /dev/null +++ b/src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "virtual-link-QKD1-QKD3"} + }, + "name": "virtual-link-QKD1-QKD3", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/dump_logs.sh b/src/tests/qkd_end2end/dump_logs.sh new file mode 100755 index 000000000..1a3dfa502 --- /dev/null +++ b/src/tests/qkd_end2end/dump_logs.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl --namespace tfs logs deployment/contextservice -c server > context.log +kubectl --namespace tfs logs deployment/deviceservice -c server > device.log +kubectl --namespace tfs logs deployment/pathcompservice -c frontend > pathcomp_front.log +kubectl --namespace tfs logs deployment/pathcompservice -c backend > pathcomp_back.log +kubectl --namespace tfs logs deployment/serviceservice -c server > service.log +kubectl --namespace tfs logs deployment/qkd-appservice -c server > qkd_app.log +kubectl --namespace tfs logs deployment/nbiservice -c server > nbi.log diff --git a/src/tests/qkd_end2end/redeploy-all.sh b/src/tests/qkd_end2end/redeploy-all.sh new file mode 100755 index 000000000..28bc684d8 --- /dev/null +++ b/src/tests/qkd_end2end/redeploy-all.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br + +# Create Docker network +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + +# Create QKD Nodes +docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test + +# Dump QKD Node Docker containers +docker ps -a + +source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh +./deploy/all.sh + +echo "Bye!" diff --git a/src/tests/qkd_end2end/deploy-qkd-nodes.sh b/src/tests/qkd_end2end/redeploy-qkd-nodes.sh similarity index 100% rename from src/tests/qkd_end2end/deploy-qkd-nodes.sh rename to src/tests/qkd_end2end/redeploy-qkd-nodes.sh -- GitLab From 4730fd4f0e1bf1f0c9bfe1dc45e96b550e10930c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 15 Jul 2025 15:05:44 +0000 Subject: [PATCH 42/82] Device component - QKD Driver: - Removed useless QKD tests - Corrected fetch of qkd node details --- src/device/.gitlab-ci.yml | 1 - src/device/service/drivers/qkd/Tools2.py | 9 +++-- .../tests/qkd/unit/test_mock_qkd_node.py | 35 ------------------- .../tests/qkd/unit/test_qkd_performance.py | 32 ----------------- 4 files changed, 7 insertions(+), 70 deletions(-) delete mode 100644 src/device/tests/qkd/unit/test_mock_qkd_node.py delete mode 100644 src/device/tests/qkd/unit/test_qkd_performance.py diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index f1355fd45..6de43d37a 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -122,7 +122,6 @@ unit_test_device: - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_mock_connectivity.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_error_handling.py" diff --git a/src/device/service/drivers/qkd/Tools2.py b/src/device/service/drivers/qkd/Tools2.py index f07a1324a..ab0a140a3 100644 --- a/src/device/service/drivers/qkd/Tools2.py +++ b/src/device/service/drivers/qkd/Tools2.py @@ -224,11 +224,16 @@ def fetch_node(url: str, resource_key: str, headers: Dict[str, str], auth: Optio try: r = requests.get(url, timeout=timeout, verify=False, auth=auth, headers=headers) r.raise_for_status() - result.append((resource_key, r.json().get('qkd_node', {}))) + data = r.json() + data.pop('qkdn_capabilities', None) + data.pop('qkd_applications', None) + data.pop('qkd_interfaces', None) + data.pop('qkd_links', None) + result.append((resource_key, data)) except requests.RequestException as e: LOGGER.error(f"Error fetching node from {url}: {e}") result.append((resource_key, e)) - + return result diff --git a/src/device/tests/qkd/unit/test_mock_qkd_node.py b/src/device/tests/qkd/unit/test_mock_qkd_node.py deleted file mode 100644 index 2eef0f6e4..000000000 --- a/src/device/tests/qkd/unit/test_mock_qkd_node.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import requests -from requests.exceptions import ConnectionError - -MOCK_QKD_ADDRESS = '127.0.0.1' -MOCK_PORT = 11111 - - -def test_mock_qkd_node_responses(): - response = requests.get(f'http://{MOCK_QKD_ADDRESS}:{MOCK_PORT}/restconf/data/etsi-qkd-sdn-node:qkd_node') - assert response.status_code == 200 - data = response.json() - assert 'qkd_node' in data - -def test_mock_node_failure_scenarios(): - try: - response = requests.get(f'http://{MOCK_QKD_ADDRESS}:12345/restconf/data/etsi-qkd-sdn-node:qkd_node') - except ConnectionError as e: - assert isinstance(e, ConnectionError) - else: - pytest.fail("ConnectionError not raised as expected") \ No newline at end of file diff --git a/src/device/tests/qkd/unit/test_qkd_performance.py b/src/device/tests/qkd/unit/test_qkd_performance.py deleted file mode 100644 index b0eafc31f..000000000 --- a/src/device/tests/qkd/unit/test_qkd_performance.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tests/unit/test_qkd_performance.py - -import pytest, time -from device.service.drivers.qkd.QKDDriver2 import QKDDriver - -MOCK_QKD_ADDRRESS = '127.0.0.1' -MOCK_PORT = 11111 - -def test_performance_under_load(): - driver = QKDDriver(address=MOCK_QKD_ADDRRESS, port=MOCK_PORT, username='user', password='pass') - driver.Connect() - - start_time = time.time() - for _ in range(1000): - driver.GetConfig(['/qkd_interfaces/qkd_interface']) - end_time = time.time() - - assert (end_time - start_time) < 60 -- GitLab From e3933acc175a9ea8ecc172aa77cd38bc8ba5bd19 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 16 Jul 2025 17:24:21 +0000 Subject: [PATCH 43/82] Common - Tools - Descriptor: - Fixed format_X_config_rules() methods when there is no config element in the descriptors. --- src/common/tools/descriptor/Tools.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index 3ca23d080..9337fd712 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -69,19 +69,19 @@ def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: def format_device_custom_config_rules(device : Dict) -> Dict: config_rules = device.get('device_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - device['device_config']['config_rules'] = config_rules + device.setdefault('device_config', {})['config_rules'] = config_rules return device def format_service_custom_config_rules(service : Dict) -> Dict: config_rules = service.get('service_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - service['service_config']['config_rules'] = config_rules + service.setdefault('service_config', {})['config_rules'] = config_rules return service def format_slice_custom_config_rules(slice_ : Dict) -> Dict: config_rules = slice_.get('slice_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - slice_['slice_config']['config_rules'] = config_rules + slice_.setdefault('slice_config', {})['config_rules'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: @@ -138,6 +138,19 @@ def link_type_to_str(link_type : Union[int, str]) -> Optional[str]: if isinstance(link_type, str): return LinkTypeEnum.Name(LinkTypeEnum.Value(link_type)) return None +LINK_TYPES_NORMAL = { + LinkTypeEnum.LINKTYPE_UNKNOWN, + LinkTypeEnum.LINKTYPE_COPPER, + LinkTypeEnum.LINKTYPE_RADIO, + LinkTypeEnum.LINKTYPE_MANAGEMENT, +} +LINK_TYPES_OPTICAL = { + LinkTypeEnum.LINKTYPE_FIBER, +} +LINK_TYPES_VIRTUAL = { + LinkTypeEnum.LINKTYPE_VIRTUAL, +} + def split_links_by_type(links : List[Dict]) -> Dict[str, List[Dict]]: typed_links = collections.defaultdict(list) for link in links: @@ -148,11 +161,11 @@ def split_links_by_type(links : List[Dict]) -> Dict[str, List[Dict]]: raise Exception(MSG.format(str(link))) link_type = LinkTypeEnum.Value(str_link_type) - if link_type in {LinkTypeEnum.LINKTYPE_UNKNOWN, LinkTypeEnum.LINKTYPE_COPPER, LinkTypeEnum.LINKTYPE_RADIO, LinkTypeEnum.LINKTYPE_MANAGEMENT}: + if link_type in LINK_TYPES_NORMAL: typed_links['normal'].append(link) - elif link_type in {LinkTypeEnum.LINKTYPE_FIBER}: + elif link_type in LINK_TYPES_OPTICAL: typed_links['optical'].append(link) - elif link_type in {LinkTypeEnum.LINKTYPE_VIRTUAL}: + elif link_type in LINK_TYPES_VIRTUAL: typed_links['virtual'].append(link) else: MSG = 'Unsupported LinkType({:s}) in Link({:s})' -- GitLab From 251f2245b8e6339e210a86a616cb5e017bfcd15c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 16 Jul 2025 17:25:27 +0000 Subject: [PATCH 44/82] NBI component - QKD connector: - Fixed parsing of qkdn_ids --- src/nbi/service/qkd_app/Resources.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/nbi/service/qkd_app/Resources.py b/src/nbi/service/qkd_app/Resources.py index 03a3d2fd2..2f391ce41 100644 --- a/src/nbi/service/qkd_app/Resources.py +++ b/src/nbi/service/qkd_app/Resources.py @@ -82,21 +82,30 @@ class CreateQKDApp(_Resource): def post(self): app = request.get_json()['app'] devices = self.context_client.ListDevices(Empty()).devices - local_device = None - local_qkdn_id = app['local_qkdn_id'] + local_qkdn_id = app.get('local_qkdn_id') + if local_qkdn_id is None: + MSG = 'local_qkdn_id not specified in qkd_app({:s})' + msg = MSG.format(str(app)) + LOGGER.exception(msg) + response = jsonify({'error': msg}) + response.status_code = HTTP_SERVERERROR + return response # This for-loop won't be necessary if Device ID is guaranteed to be the same as QKDN Id + local_device = None for device in devices: for config_rule in device.device_config.config_rules: - if config_rule.custom.resource_key == '__node__': - value = json.loads(config_rule.custom.resource_value) - if local_qkdn_id != value['qkdn_id']: continue - local_device = device - break + if config_rule.custom.resource_key != '__node__': continue + value = json.loads(config_rule.custom.resource_value) + qkdn_id = value.get('qkdn_id') + if qkdn_id is None: continue + if local_qkdn_id != qkdn_id: continue + local_device = device + break if local_device is None: - MSG = 'Unable to find local_device for local_qkdn_id({:s})' + MSG = 'Unable to find device for local_qkdn_id({:s})' msg = MSG.format(str(local_qkdn_id)) LOGGER.exception(msg) response = jsonify({'error': msg}) -- GitLab From 281a132d3be8c6b3469de8de29f219f63c050889 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 16 Jul 2025 17:26:27 +0000 Subject: [PATCH 45/82] QKD App: - Fixed UUIDs of qkdn_ids in test_create_apps - Fixed reply dumps in test_create_apps --- src/qkd_app/.gitlab-ci.yml | 2 +- src/qkd_app/tests/test_create_apps.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml index 573065b9e..09fac2d42 100644 --- a/src/qkd_app/.gitlab-ci.yml +++ b/src/qkd_app/.gitlab-ci.yml @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/src/qkd_app/tests/test_create_apps.py b/src/qkd_app/tests/test_create_apps.py index 82394e862..c16c31906 100644 --- a/src/qkd_app/tests/test_create_apps.py +++ b/src/qkd_app/tests/test_create_apps.py @@ -22,19 +22,21 @@ QKD_REQUEST_1 = { 'server_app_id': '1', 'client_app_id': [], 'app_status': 'ON', - 'local_qkdn_id': '00000001-0000-0000-0000-0000000000', - 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + 'local_qkdn_id': '00000001-0000-0000-0000-000000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-000000000000'], } } -print(requests.post(QKD_URL, json=QKD_REQUEST_1)) +reply = requests.post(QKD_URL, json=QKD_REQUEST_1) +print(reply.status_code, reply.text) QKD_REQUEST_2 = { 'app': { 'server_app_id': '1', 'client_app_id': [], 'app_status': 'ON', - 'local_qkdn_id': '00000003-0000-0000-0000-0000000000', - 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + 'local_qkdn_id': '00000003-0000-0000-0000-000000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-000000000000'], } } -print(requests.post(QKD_URL, json=QKD_REQUEST_2)) +reply = requests.post(QKD_URL, json=QKD_REQUEST_2) +print(reply.status_code, reply.text) -- GitLab From 80053007cb338eb605e99b04781d56d46fafc149 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 16 Jul 2025 17:30:11 +0000 Subject: [PATCH 46/82] QKD E2E Test: - Conveniently renamed descriptor files - Defined scripts and python test code - Updated Dockerfile and requirements.in - Added convenience microk8s status wait ready in redeploy-all script - Added QKD App service in log dump script - Updated QKD_E2E GitlabCI/CD pipeline descriptor - Updated README.md and merged commands.txt on it --- src/tests/qkd_end2end/.gitignore | 14 ++ src/tests/qkd_end2end/.gitlab-ci.yml | 178 +++++++++---- src/tests/qkd_end2end/Dockerfile | 86 +++++++ src/tests/qkd_end2end/README.md | 48 ++++ src/tests/qkd_end2end/__init__.py | 14 ++ src/tests/qkd_end2end/commands.txt | 8 - ...topology_qkd.json => tfs-01-topology.json} | 0 ...json => tfs-02-direct-link-qkd1-qkd2.json} | 0 ...json => tfs-03-direct-link-qkd2-qkd3.json} | 0 ...son => tfs-04-virtual-link-qkd1-qkd3.json} | 0 .../data/tfs-05-app-1-qkd1-qkd3.json | 9 + .../data/tfs-06-app-1-qkd3-qkd1.json | 9 + src/tests/qkd_end2end/dump_logs.sh | 1 + src/tests/qkd_end2end/redeploy-all.sh | 5 + src/tests/qkd_end2end/requirements.in | 15 ++ .../qkd_end2end/scripts/run-01-onboarding.sh | 20 ++ .../scripts/run-02-create-links.sh | 20 ++ .../scripts/run-03-create-external-app.sh | 20 ++ .../scripts/run-04-delete-external-app.sh | 20 ++ .../scripts/run-05-delete-links.sh | 20 ++ .../qkd_end2end/scripts/run-06-cleanup.sh | 20 ++ src/tests/qkd_end2end/tests/Fixtures.py | 43 ++++ src/tests/qkd_end2end/tests/Tools.py | 109 ++++++++ src/tests/qkd_end2end/tests/__init__.py | 14 ++ .../qkd_end2end/tests/test_01_onboarding.py | 67 +++++ .../qkd_end2end/tests/test_02_create_links.py | 235 ++++++++++++++++++ .../tests/test_03_create_external_app.py | 123 +++++++++ .../tests/test_04_delete_external_app.py | 118 +++++++++ .../qkd_end2end/tests/test_05_delete_links.py | 134 ++++++++++ .../qkd_end2end/tests/test_06_cleanup.py | 44 ++++ 30 files changed, 1344 insertions(+), 50 deletions(-) create mode 100644 src/tests/qkd_end2end/.gitignore create mode 100644 src/tests/qkd_end2end/Dockerfile create mode 100644 src/tests/qkd_end2end/README.md create mode 100644 src/tests/qkd_end2end/__init__.py delete mode 100644 src/tests/qkd_end2end/commands.txt rename src/tests/qkd_end2end/data/{topology_qkd.json => tfs-01-topology.json} (100%) rename src/tests/qkd_end2end/data/{direct-link-qkd1-qkd2.json => tfs-02-direct-link-qkd1-qkd2.json} (100%) rename src/tests/qkd_end2end/data/{direct-link-qkd2-qkd3.json => tfs-03-direct-link-qkd2-qkd3.json} (100%) rename src/tests/qkd_end2end/data/{virtual-link-qkd1-qkd3.json => tfs-04-virtual-link-qkd1-qkd3.json} (100%) create mode 100644 src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json create mode 100644 src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json create mode 100644 src/tests/qkd_end2end/requirements.in create mode 100755 src/tests/qkd_end2end/scripts/run-01-onboarding.sh create mode 100755 src/tests/qkd_end2end/scripts/run-02-create-links.sh create mode 100755 src/tests/qkd_end2end/scripts/run-03-create-external-app.sh create mode 100755 src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh create mode 100755 src/tests/qkd_end2end/scripts/run-05-delete-links.sh create mode 100755 src/tests/qkd_end2end/scripts/run-06-cleanup.sh create mode 100644 src/tests/qkd_end2end/tests/Fixtures.py create mode 100644 src/tests/qkd_end2end/tests/Tools.py create mode 100644 src/tests/qkd_end2end/tests/__init__.py create mode 100644 src/tests/qkd_end2end/tests/test_01_onboarding.py create mode 100644 src/tests/qkd_end2end/tests/test_02_create_links.py create mode 100644 src/tests/qkd_end2end/tests/test_03_create_external_app.py create mode 100644 src/tests/qkd_end2end/tests/test_04_delete_external_app.py create mode 100644 src/tests/qkd_end2end/tests/test_05_delete_links.py create mode 100644 src/tests/qkd_end2end/tests/test_06_cleanup.py diff --git a/src/tests/qkd_end2end/.gitignore b/src/tests/qkd_end2end/.gitignore new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/tests/qkd_end2end/.gitignore @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 074f6b9ea..6e56f3d0c 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -37,6 +37,7 @@ build qkd_end2end: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test qkd_end2end: + timeout: 30m variables: TEST_NAME: 'qkd_end2end' stage: end2end_test @@ -68,59 +69,156 @@ end2end_test qkd_end2end: --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" mock-qkd-node:test + # Dump configuration of the QKD Nodes (script, before any configuration) + - echo "[QKD-NODE-01] Config initial:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config initial:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config initial:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + # Check MicroK8s is ready - microk8s status --wait-ready - kubectl get pods --all-namespaces + # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/qkd_appservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + # Deploy TeraFlowSDN + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/kafka.sh + #- ./deploy/qdb.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + ## Wait for Context to be subscribed to NATS + ## WARNING: this loop is infinite if there is no subscriber (such as monitoring). + ## Investigate if we can use a counter to limit the number of iterations. + ## For now, keep it commented out. + #- LOOP_MAX_ATTEMPTS=180 + #- LOOP_COUNTER=0 + #- > + # while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do + # echo "Attempt: $LOOP_COUNTER" + # kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1; + # sleep 1; + # LOOP_COUNTER=$((LOOP_COUNTER + 1)) + # if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + # echo "Max attempts reached, exiting the loop." + # break + # fi + # done + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - # Wait for CockroachDB and NATS to initialize - - echo "Waiting for CockroachDB to be ready..." - - while ! docker logs crdb 2>&1 | grep -q 'CockroachDB node starting'; do sleep 1; done - - docker logs crdb - - echo "Waiting for NATS to be ready..." - - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - docker logs nats + # Run end-to-end test: onboard scenario + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-01-onboarding.sh - # Start mock QKD nodes - - docker run --name mock_qkd -d --network=teraflowbridge -v "$PWD/src/tests/tools/mock_qkd_nodes:/app" python:3.9-slim bash -c "cd /app && ./start.sh" + # Run end-to-end test: create QKD links + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-02-create-links.sh - # Wait for mock QKD nodes to initialize - - echo "Waiting for mock QKD nodes to be ready..." - - sleep 10 + # Dump configuration of the QKD Nodes (script, after create QKD links) + - echo "[QKD-NODE-01] Config with links:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo - # Deploy TeraFlowSDN services - - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + # TODO: check config of QKD Nodes with created links is correct - # Deploy Context Service - - docker run --name context -d -p 1010:1010 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --env "MB_BACKEND=nats" --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" --network=teraflowbridge $CI_REGISTRY_IMAGE/context:latest - - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + # Run end-to-end test: create external app + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-03-create-external-app.sh - # Deploy Device Service - - docker run --name device -d -p 2020:2020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --network=teraflowbridge $CI_REGISTRY_IMAGE/device:latest - - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + # Dump configuration of the QKD Nodes (script, after create external app) + - echo "[QKD-NODE-01] Config with links and external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links and external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links and external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo - # Deploy PathComp Services (frontend and backend) - - docker run --name pathcomp-backend -d -p 8081:8081 --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-backend:latest - - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - docker run --name pathcomp-frontend -d -p 10020:10020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" --env "PATHCOMP_BACKEND_PORT=8081" --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-frontend:latest - - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + # TODO: check config of QKD Nodes with created links and external app is correct - # Deploy Service Component - - docker run --name service -d -p 3030:3030 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" --volume "$PWD/src/service/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/service:latest + # Run end-to-end test: delete external app + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-04-delete-external-app.sh - # Wait for services to initialize - - sleep 10 + # Dump configuration of the QKD Nodes (script, after delete external app) + - echo "[QKD-NODE-01] Config with links and deleted external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links and deleted external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links and deleted external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo - # Run end-to-end tests for QKD application + # TODO: check config of QKD Nodes with created links and deleted external app is correct + + # Run end-to-end test: delete QKD links - > - docker run -t --rm --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" - $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-05-delete-links.sh + + # Dump configuration of the QKD Nodes (script, after delete QKD links) + - echo "[QKD-NODE-01] Config with deleted links and deleted external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with deleted links and deleted external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with deleted links and deleted external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # TODO: check config of QKD Nodes with deleted links and deleted external app is correct + + # Run end-to-end test: cleanup scenario + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-06-cleanup.sh after_script: # Dump configuration of the QKD Nodes (after_script) @@ -139,13 +237,6 @@ end2end_test qkd_end2end: - docker logs qkd-node-02 - docker logs qkd-node-03 - # Dump logs for TeraFlowSDN components - - docker logs context - - docker logs device - - docker logs pathcomp-frontend - - docker logs pathcomp-backend - - docker logs service - # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -156,10 +247,13 @@ end2end_test qkd_end2end: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qkd-appservice -c server # Clean up - - docker rm -f context device pathcomp-frontend pathcomp-backend service mock_qkd crdb nats - - docker network rm teraflowbridge + - docker rm -f ${TEST_NAME} || true + - kubectl delete namespaces tfs || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br - docker volume prune --force - docker image prune --force + # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/qkd_end2end/Dockerfile b/src/tests/qkd_end2end/Dockerfile new file mode 100644 index 000000000..c260c90e4 --- /dev/null +++ b/src/tests/qkd_end2end/Dockerfile @@ -0,0 +1,86 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/qkd_end2end +WORKDIR /var/teraflow/tests/qkd_end2end +COPY src/tests/qkd_end2end/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/vnt_manager/__init__.py vnt_manager/__init__.py +COPY src/vnt_manager/client/. vnt_manager/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/qkd_end2end/__init__.py ./tests/qkd_end2end/__init__.py +COPY src/tests/qkd_end2end/data/. ./tests/qkd_end2end/data/ +COPY src/tests/qkd_end2end/tests/. ./tests/qkd_end2end/tests/ +COPY src/tests/qkd_end2end/scripts/. ./ + +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install tree && \ + rm -rf /var/lib/apt/lists/* + +RUN tree -la /var/teraflow diff --git a/src/tests/qkd_end2end/README.md b/src/tests/qkd_end2end/README.md new file mode 100644 index 000000000..f00ef4ee1 --- /dev/null +++ b/src/tests/qkd_end2end/README.md @@ -0,0 +1,48 @@ +# QKD End-to-End Test + +## Emulated QKD Nodes +See `src/tests/tools/mock_qkd_node`. +Here we deploy 3 emulated QKD Nodes initialized with configurations `data/qkd-node-XX.json`. + +### (Re-)Deploy the 3 QKD Nodes +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/redeploy-qkd-nodes.sh +``` + +### Check their configuration +```bash +curl http://:/restconf/data/etsi-qkd-sdn-node: +``` + +### Update their configuration using root path +```bash +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://:/restconf/data/etsi-qkd-sdn-node: +``` + +### Update their configuration using sub-entity path +```bash +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://:/restconf/data/etsi-qkd-sdn-node:qkd_node +``` + +### Destroy scenario +```bash +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br +``` + +## TeraFlowSDN Deployment +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/redeploy-tfs.sh +``` + +### QKD Node Topology +The topology descriptor for the QKD nodes is: `data/tfs-01-topology.json` + +### Dump TFS component logs to files +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/dump_logs.sh +``` +Will create files `.log` diff --git a/src/tests/qkd_end2end/__init__.py b/src/tests/qkd_end2end/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/tests/qkd_end2end/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/commands.txt b/src/tests/qkd_end2end/commands.txt deleted file mode 100644 index 36fb03cdf..000000000 --- a/src/tests/qkd_end2end/commands.txt +++ /dev/null @@ -1,8 +0,0 @@ - -src/tests/qkd_end2end/redeploy-all.sh -src/tests/qkd_end2end/redeploy-tfs.sh -src/tests/qkd_end2end/redeploy-qkd-nodes.sh - -python src/qkd_app/tests/test_create_apps.py - -src/tests/qkd_end2end/dump_logs.sh diff --git a/src/tests/qkd_end2end/data/topology_qkd.json b/src/tests/qkd_end2end/data/tfs-01-topology.json similarity index 100% rename from src/tests/qkd_end2end/data/topology_qkd.json rename to src/tests/qkd_end2end/data/tfs-01-topology.json diff --git a/src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json b/src/tests/qkd_end2end/data/tfs-02-direct-link-qkd1-qkd2.json similarity index 100% rename from src/tests/qkd_end2end/data/direct-link-qkd1-qkd2.json rename to src/tests/qkd_end2end/data/tfs-02-direct-link-qkd1-qkd2.json diff --git a/src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json b/src/tests/qkd_end2end/data/tfs-03-direct-link-qkd2-qkd3.json similarity index 100% rename from src/tests/qkd_end2end/data/direct-link-qkd2-qkd3.json rename to src/tests/qkd_end2end/data/tfs-03-direct-link-qkd2-qkd3.json diff --git a/src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json b/src/tests/qkd_end2end/data/tfs-04-virtual-link-qkd1-qkd3.json similarity index 100% rename from src/tests/qkd_end2end/data/virtual-link-qkd1-qkd3.json rename to src/tests/qkd_end2end/data/tfs-04-virtual-link-qkd1-qkd3.json diff --git a/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json b/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json new file mode 100644 index 000000000..95fcf24e0 --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json @@ -0,0 +1,9 @@ +{ + "app": { + "server_app_id": "1", + "client_app_id": [], + "app_status": "ON", + "local_qkdn_id": "00000001-0000-0000-0000-000000000000", + "backing_qkdl_id": ["00000003-0002-0000-0000-000000000000"] + } +} diff --git a/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json b/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json new file mode 100644 index 000000000..1e76a317d --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json @@ -0,0 +1,9 @@ +{ + "app": { + "server_app_id": "1", + "client_app_id": [], + "app_status": "ON", + "local_qkdn_id": "00000003-0000-0000-0000-000000000000", + "backing_qkdl_id": ["00000003-0002-0000-0000-000000000000"] + } +} diff --git a/src/tests/qkd_end2end/dump_logs.sh b/src/tests/qkd_end2end/dump_logs.sh index 1a3dfa502..bde1f4720 100755 --- a/src/tests/qkd_end2end/dump_logs.sh +++ b/src/tests/qkd_end2end/dump_logs.sh @@ -7,3 +7,4 @@ kubectl --namespace tfs logs deployment/pathcompservice -c backend > pathcomp_ba kubectl --namespace tfs logs deployment/serviceservice -c server > service.log kubectl --namespace tfs logs deployment/qkd-appservice -c server > qkd_app.log kubectl --namespace tfs logs deployment/nbiservice -c server > nbi.log +kubectl --namespace tfs logs deployment/webuiservice -c server > webui.log diff --git a/src/tests/qkd_end2end/redeploy-all.sh b/src/tests/qkd_end2end/redeploy-all.sh index 28bc684d8..a1182de23 100755 --- a/src/tests/qkd_end2end/redeploy-all.sh +++ b/src/tests/qkd_end2end/redeploy-all.sh @@ -33,6 +33,11 @@ docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.10 # Dump QKD Node Docker containers docker ps -a +echo + +# Wait till MicroK8s is stabilized... +microk8s status --wait-ready +echo source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh ./deploy/all.sh diff --git a/src/tests/qkd_end2end/requirements.in b/src/tests/qkd_end2end/requirements.in new file mode 100644 index 000000000..5c92783a2 --- /dev/null +++ b/src/tests/qkd_end2end/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +requests==2.27.* diff --git a/src/tests/qkd_end2end/scripts/run-01-onboarding.sh b/src/tests/qkd_end2end/scripts/run-01-onboarding.sh new file mode 100755 index 000000000..df9186204 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-01-onboarding.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_onboarding.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_01_onboarding.py diff --git a/src/tests/qkd_end2end/scripts/run-02-create-links.sh b/src/tests/qkd_end2end/scripts/run-02-create-links.sh new file mode 100755 index 000000000..806d2ff11 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-02-create-links.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_create_links.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_02_create_links.py diff --git a/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh b/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh new file mode 100755 index 000000000..45dcaf27e --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_ietf_create.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_03_create_external_app.py diff --git a/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh b/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh new file mode 100755 index 000000000..e988042a5 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_ietf_remove.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_04_delete_external_app.py diff --git a/src/tests/qkd_end2end/scripts/run-05-delete-links.sh b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh new file mode 100755 index 000000000..f00a8711f --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_tfs_remove.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_05-delete-links.py diff --git a/src/tests/qkd_end2end/scripts/run-06-cleanup.sh b/src/tests/qkd_end2end/scripts/run-06-cleanup.sh new file mode 100755 index 000000000..991a5325e --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-06-cleanup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_cleanup.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_06_cleanup.py diff --git a/src/tests/qkd_end2end/tests/Fixtures.py b/src/tests/qkd_end2end/tests/Fixtures.py new file mode 100644 index 000000000..788e29d59 --- /dev/null +++ b/src/tests/qkd_end2end/tests/Fixtures.py @@ -0,0 +1,43 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient + +@pytest.fixture(scope='session') +def context_client(): + _client = ContextClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def device_client(): + _client = DeviceClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def qkd_app_client(): + _client = QKDAppClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def service_client(): + _client = ServiceClient() + yield _client + _client.close() diff --git a/src/tests/qkd_end2end/tests/Tools.py b/src/tests/qkd_end2end/tests/Tools.py new file mode 100644 index 000000000..bbee845cd --- /dev/null +++ b/src/tests/qkd_end2end/tests/Tools.py @@ -0,0 +1,109 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum, logging, requests +from typing import Any, Dict, List, Optional, Set, Union +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_http + +NBI_ADDRESS = get_service_host(ServiceNameEnum.NBI) +NBI_PORT = get_service_port_http(ServiceNameEnum.NBI) +NBI_USERNAME = 'admin' +NBI_PASSWORD = 'admin' +NBI_BASE_URL = '' + +class RestRequestMethod(enum.Enum): + GET = 'get' + POST = 'post' + PUT = 'put' + PATCH = 'patch' + DELETE = 'delete' + +EXPECTED_STATUS_CODES : Set[int] = { + requests.codes['OK' ], + requests.codes['CREATED' ], + requests.codes['ACCEPTED' ], + requests.codes['NO_CONTENT'], +} + +def do_rest_request( + method : RestRequestMethod, url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + request_url = 'http://{:s}:{:s}@{:s}:{:d}{:s}{:s}'.format( + NBI_USERNAME, NBI_PASSWORD, NBI_ADDRESS, NBI_PORT, str(NBI_BASE_URL), url + ) + + if logger is not None: + msg = 'Request: {:s} {:s}'.format(str(method.value).upper(), str(request_url)) + if body is not None: msg += ' body={:s}'.format(str(body)) + logger.warning(msg) + reply = requests.request(method.value, request_url, timeout=timeout, json=body, allow_redirects=allow_redirects) + if logger is not None: + logger.warning('Reply: {:s}'.format(str(reply.text))) + assert reply.status_code in expected_status_codes, 'Reply failed with status code {:d}'.format(reply.status_code) + + if reply.content and len(reply.content) > 0: return reply.json() + return None + +def do_rest_get_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.GET, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_post_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.POST, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_put_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PUT, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_patch_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PATCH, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_delete_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.DELETE, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) diff --git a/src/tests/qkd_end2end/tests/__init__.py b/src/tests/qkd_end2end/tests/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/tests/qkd_end2end/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/tests/test_01_onboarding.py b/src/tests/qkd_end2end/tests/test_01_onboarding.py new file mode 100644 index 000000000..763d7da17 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_01_onboarding.py @@ -0,0 +1,67 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_onboarding( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 + assert num_devices_enabled == num_devices diff --git a/src/tests/qkd_end2end/tests/test_02_create_links.py b/src/tests/qkd_end2end/tests/test_02_create_links.py new file mode 100644 index 000000000..fb8a91342 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_02_create_links.py @@ -0,0 +1,235 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, Empty, ServiceStatusEnum, ServiceTypeEnum +from common.proto.qkd_app_pb2 import QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient +from .Fixtures import ( # pylint: disable=unused-import + context_client, device_client, service_client, qkd_app_client +) + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +def compose_path(file_name : str) -> str: + return os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', file_name) + + +def test_check_qkd_apps_before( + context_client : ContextClient, # pylint: disable=redefined-outer-name + qkd_app_client : QKDAppClient, # pylint: disable=redefined-outer-name +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 0 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 0 + +def test_create_direct_link_qkd1_qkd2( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-direct-link-qkd1-qkd2.json') + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 1 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_create_direct_link_qkd2_qkd3( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-direct-link-qkd2-qkd3.json') + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 2 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 2 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_create_virtual_link_qkd1_qkd3( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-virtual-link-qkd1-qkd3.json'), + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 3 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_check_qkd_apps_after( + context_client : ContextClient, # pylint: disable=redefined-outer-name + qkd_app_client : QKDAppClient, # pylint: disable=redefined-outer-name +): + response = context_client.ListDevices(Empty()) + LOGGER.warning('Devices[{:d}] = {:s}'.format( + len(response.devices), grpc_message_to_json_string(response) + )) + device_uuid_to_name = { + device.device_id.device_uuid.uuid : device.name + for device in response.devices + } + + device_qkd1_uuid = device_uuid_to_name.get('QKD1') + assert device_qkd1_uuid is not None + + device_qkd3_uuid = device_uuid_to_name.get('QKD3') + assert device_qkd3_uuid is not None + + pending_device_pairs = { + (device_qkd1_uuid, device_qkd3_uuid), + (device_qkd3_uuid, device_qkd1_uuid), + } + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + for app in response.apps: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + assert app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL + local_device_id = app.local_device_id.device_uuid.uuid + remote_device_id = app.remote_device_id.device_uuid.uuid + device_pair = (local_device_id, remote_device_id) + assert device_pair in pending_device_pairs + pending_device_pairs.pop(device_pair) + + assert len(pending_device_pairs) == 0 diff --git a/src/tests/qkd_end2end/tests/test_03_create_external_app.py b/src/tests/qkd_end2end/tests/test_03_create_external_app.py new file mode 100644 index 000000000..fed2370ff --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_03_create_external_app.py @@ -0,0 +1,123 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json, logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.proto.qkd_app_pb2 import QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from .Fixtures import context_client, qkd_app_client # pylint: disable=unused-import +from .Tools import do_rest_post_request + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +def compose_path(file_name : str) -> str: + return os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', file_name) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + for app in response.apps: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + assert app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL + + +# pylint: disable=redefined-outer-name +def test_create_external_app_qkd1_to_qkd3( + qkd_app_client : QKDAppClient, +): + request_file = compose_path('tfs-05-app-1-qkd1-qkd3.json') + + # Issue external QKD App creation request (QKD1-QKD3) + with open(request_file, 'r', encoding='UTF-8') as f: + req_data = json.load(f) + + URL = '/qkd_app/create_qkd_app' + do_rest_post_request(URL, body=req_data, logger=LOGGER, expected_status_codes={200}) + + # Verify QKD app was created + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 + + +# pylint: disable=redefined-outer-name +def test_create_external_app_qkd3_to_qkd1( + qkd_app_client : QKDAppClient, +): + request_file = compose_path('tfs-05-app-1-qkd3-qkd1.json') + + # Issue external QKD App creation request (QKD3-QKD1) + with open(request_file, 'r', encoding='UTF-8') as f: + req_data = json.load(f) + + URL = '/qkd_app/create_qkd_app' + do_rest_post_request(URL, body=req_data, logger=LOGGER, expected_status_codes={200}) + + # Verify no new QKD app was created + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 diff --git a/src/tests/qkd_end2end/tests/test_04_delete_external_app.py b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py new file mode 100644 index 000000000..8c8236e6e --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py @@ -0,0 +1,118 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.proto.qkd_app_pb2 import AppId, QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from .Fixtures import context_client, qkd_app_client # pylint: disable=unused-import + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are 3 services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 + + +# pylint: disable=redefined-outer-name +def test_delete_external_app( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + external_app_uuids : Set[str] = set() + for app in response.apps: + if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_CLIENT: continue + external_app_uuids.add(app.app_id.app_uuid.uuid) + + # Identify QKD ext app to delete + assert len(external_app_uuids) == 1 + external_app_uuid = set(external_app_uuids).pop() + + app_id = AppId() + app_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + app_id.app_uuid.uuid = external_app_uuid + + response = qkd_app_client.DeleteApp(app_id) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_after( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + num_external += 1 + assert num_internal == 2 + assert num_external == 0 diff --git a/src/tests/qkd_end2end/tests/test_05_delete_links.py b/src/tests/qkd_end2end/tests/test_05_delete_links.py new file mode 100644 index 000000000..b04236d47 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_05_delete_links.py @@ -0,0 +1,134 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceId +from common.proto.qkd_app_pb2 import AppId, QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient +from .Fixtures import ( + # pylint: disable=unused-import + context_client, qkd_app_client, service_client +) + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 0 + + +# pylint: disable=redefined-outer-name +def test_delete_internal_apps( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + # Identify internal QKD apps to delete + internal_app_uuids : Set[str] = set() + for app in response.apps: + if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_CLIENT: continue + internal_app_uuids.add(app.app_id.app_uuid.uuid) + + assert len(internal_app_uuids) == 2 + for app_uuid in internal_app_uuids: + app_id = AppId() + app_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + app_id.app_uuid.uuid = app_uuid + response = qkd_app_client.DeleteApp(app_id) + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 0 + + +# pylint: disable=redefined-outer-name +def test_delete_services_associated_qkd_apps( + context_client : ContextClient, + service_client : ServiceClient, +): + # Check there are 3 services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + # Identify services for QKD links to delete + virtual_link_service_uuids : Set[str] = set() + direct_link_service_uuids : Set[str] = set() + for service in response.services: + if 'virtual' in str(service.name).lower(): + virtual_link_service_uuids.add(service.service_id.service_uuid.uuid) + if 'direct' in str(service.name).lower(): + direct_link_service_uuids.add(service.service_id.service_uuid.uuid) + + assert len(virtual_link_service_uuids) == 1 + assert len(direct_link_service_uuids ) == 2 + + # Delete the services + for svc_uuid in virtual_link_service_uuids: + svc_id = ServiceId() + svc_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + svc_id.service_uuid.uuid = svc_uuid + response = service_client.DeleteService(svc_id) + + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 0 diff --git a/src/tests/qkd_end2end/tests/test_06_cleanup.py b/src/tests/qkd_end2end/tests/test_06_cleanup.py new file mode 100644 index 000000000..20afb5fe0 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_06_cleanup.py @@ -0,0 +1,44 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.tools.descriptor.Loader import DescriptorLoader, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_cleanup( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) -- GitLab From 300b8367c429fe30058ed90f26b539cf3fd972e4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 08:44:43 +0000 Subject: [PATCH 47/82] Mock QKD Node: - Corrected GitLab CI/CD pipeline - Corrected build.sh script - Corrected Dockerfile --- src/tests/tools/mock_qkd_node/.gitlab-ci.yml | 4 ++-- src/tests/tools/mock_qkd_node/Dockerfile | 3 --- src/tests/tools/mock_qkd_node/build.sh | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml index 787f27072..641169a88 100644 --- a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml +++ b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml @@ -18,7 +18,7 @@ build mock_qkd_node: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - - docker buildx build -t "$CI_REGISTRY_IMAGE/mock_qkd_node:test" -f ./src/tests/tools/mock_qkd_node/Dockerfile . + - docker buildx build -t "$CI_REGISTRY_IMAGE/mock-qkd-node:test" -f ./src/tests/tools/mock_qkd_node/Dockerfile ./src/tests/tools/mock_qkd_node - docker push "$CI_REGISTRY_IMAGE/mock_qkd_node:test" after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi @@ -28,7 +28,7 @@ build mock_qkd_node: - changes: - src/common/**/*.py - proto/*.proto - - src/src/tests/tools/mock_qkd_node/**/*.{py,in,yml,yaml} + - src/src/tests/tools/mock_qkd_node/**/*.{py,in,yml,yaml,yang,sh,json} - src/src/tests/tools/mock_qkd_node/Dockerfile - src/device/**/*.{py,in,yml} - src/device/Dockerfile diff --git a/src/tests/tools/mock_qkd_node/Dockerfile b/src/tests/tools/mock_qkd_node/Dockerfile index ef53aed75..5e39eb3c0 100644 --- a/src/tests/tools/mock_qkd_node/Dockerfile +++ b/src/tests/tools/mock_qkd_node/Dockerfile @@ -26,9 +26,6 @@ RUN fc-lang-install -v # Create component sub-folders, and copy content RUN mkdir -p /var/mock_qkd_node/ WORKDIR /var/mock_qkd_node -COPY yang/. yang/ - -# Copy code and data COPY ./yang ./yang COPY ./startup.json ./startup.json COPY ./qkd_node.py ./qkd_node.py diff --git a/src/tests/tools/mock_qkd_node/build.sh b/src/tests/tools/mock_qkd_node/build.sh index d2b102209..e18dd9863 100755 --- a/src/tests/tools/mock_qkd_node/build.sh +++ b/src/tests/tools/mock_qkd_node/build.sh @@ -16,6 +16,6 @@ # Make folder containing the script the root folder for its execution cd $(dirname $0) -docker build -t mock-qkd-node:test -f Dockerfile . +docker buildx build -t mock-qkd-node:test -f Dockerfile . #docker tag mock-qkd-node:test localhost:32000/tfs/mock-qkd-node:test #docker push localhost:32000/tfs/mock-qkd-node:test -- GitLab From 439c79461e96461bcbeebd6ddb12cd99f3d6b768 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 08:55:07 +0000 Subject: [PATCH 48/82] GitLab CI/CD pipeline: - Corrected image names in mock_qkd_node - Corrected image names in qkd_end2end --- src/tests/qkd_end2end/.gitlab-ci.yml | 7 ++++--- src/tests/tools/mock_qkd_node/.gitlab-ci.yml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 6e56f3d0c..f059d3c7f 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -53,21 +53,22 @@ end2end_test qkd_end2end: script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + - docker pull "${CI_REGISTRY_IMAGE}/mock-qkd-node:test" # Deploy scenario with mock QKD Nodes - docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br - > docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 --volume "$PWD/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" - mock-qkd-node:test + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - > docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 --volume "$PWD/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" - mock-qkd-node:test + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - > docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" - mock-qkd-node:test + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test # Dump configuration of the QKD Nodes (script, before any configuration) - echo "[QKD-NODE-01] Config initial:" diff --git a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml index 641169a88..593283591 100644 --- a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml +++ b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml @@ -19,7 +19,7 @@ build mock_qkd_node: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - docker buildx build -t "$CI_REGISTRY_IMAGE/mock-qkd-node:test" -f ./src/tests/tools/mock_qkd_node/Dockerfile ./src/tests/tools/mock_qkd_node - - docker push "$CI_REGISTRY_IMAGE/mock_qkd_node:test" + - docker push "$CI_REGISTRY_IMAGE/mock-qkd-node:test" after_script: - docker images --filter="dangling=true" --quiet | xargs -r docker rmi rules: -- GitLab From f34d04be8fd4e9fde8af21ce332ba9bca14c2577 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 08:59:40 +0000 Subject: [PATCH 49/82] QKD E2E Test: - Corrected paths in GitlabCI/CD pipeline descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index f059d3c7f..975ff943b 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -59,15 +59,15 @@ end2end_test qkd_end2end: - docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br - > docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 - --volume "$PWD/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - > docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 - --volume "$PWD/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - > docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 - --volume "$PWD/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" ${CI_REGISTRY_IMAGE}/mock-qkd-node:test # Dump configuration of the QKD Nodes (script, before any configuration) -- GitLab From 94caa9f56900e91f19e37e592231a3b558932448 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 09:04:42 +0000 Subject: [PATCH 50/82] QKD E2E Test: - Corrected GitlabCI/CD pipeline descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 975ff943b..e2cd4cdb0 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -70,6 +70,11 @@ end2end_test qkd_end2end: --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" ${CI_REGISTRY_IMAGE}/mock-qkd-node:test + # Dump logs of the QKD Nodes (script, before any configuration) + - docker logs qkd-node-01 + - docker logs qkd-node-02 + - docker logs qkd-node-03 + # Dump configuration of the QKD Nodes (script, before any configuration) - echo "[QKD-NODE-01] Config initial:" - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" @@ -222,6 +227,11 @@ end2end_test qkd_end2end: $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-06-cleanup.sh after_script: + # Dump logs of the QKD Nodes (after_script) + - docker logs qkd-node-01 + - docker logs qkd-node-02 + - docker logs qkd-node-03 + # Dump configuration of the QKD Nodes (after_script) - echo "[QKD-NODE-01] Config after_script:" - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" @@ -233,11 +243,6 @@ end2end_test qkd_end2end: - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" - echo - # Dump logs of the QKD Nodes (after_script) - - docker logs qkd-node-01 - - docker logs qkd-node-02 - - docker logs qkd-node-03 - # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server -- GitLab From 8773904de7b847829796feb7224982b5bdeccd28 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 09:18:20 +0000 Subject: [PATCH 51/82] QKD E2E Test: - Corrected GitlabCI/CD pipeline descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index e2cd4cdb0..e52c9da0a 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -70,6 +70,11 @@ end2end_test qkd_end2end: --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" ${CI_REGISTRY_IMAGE}/mock-qkd-node:test + - echo "Waiting for QKD Nodes to initialize..." + - while ! docker logs qkd-node-01 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - while ! docker logs qkd-node-02 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - while ! docker logs qkd-node-03 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + # Dump logs of the QKD Nodes (script, before any configuration) - docker logs qkd-node-01 - docker logs qkd-node-02 -- GitLab From 92ea563d5f5517f7d1794b08dd026945f93ae766 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 09:22:09 +0000 Subject: [PATCH 52/82] QKD E2E Test: - Corrected GitlabCI/CD pipeline descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index e52c9da0a..6f1fc65b0 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -71,9 +71,9 @@ end2end_test qkd_end2end: ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - echo "Waiting for QKD Nodes to initialize..." - - while ! docker logs qkd-node-01 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done - - while ! docker logs qkd-node-02 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done - - while ! docker logs qkd-node-03 2>&1 | grep -q "WARNING\: All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - while ! docker logs qkd-node-01 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - while ! docker logs qkd-node-02 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - while ! docker logs qkd-node-03 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done # Dump logs of the QKD Nodes (script, before any configuration) - docker logs qkd-node-01 -- GitLab From 971b02863436382cf01197de131ad472f9314b2d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 11:02:41 +0000 Subject: [PATCH 53/82] QKD E2E Test: - Corrected GitlabCI/CD pipeline descriptor --- src/tests/qkd_end2end/.gitlab-ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 6f1fc65b0..0b66487f0 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -71,9 +71,21 @@ end2end_test qkd_end2end: ${CI_REGISTRY_IMAGE}/mock-qkd-node:test - echo "Waiting for QKD Nodes to initialize..." - - while ! docker logs qkd-node-01 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done - - while ! docker logs qkd-node-02 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done - - while ! docker logs qkd-node-03 2>&1 | grep -q "All log messages before absl\:\:InitializeLog\(\) is called are written to STDERR"; do sleep 1; done + - > + while ! docker logs qkd-node-01 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done + - > + while ! docker logs qkd-node-02 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done + - > + while ! docker logs qkd-node-03 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done # Dump logs of the QKD Nodes (script, before any configuration) - docker logs qkd-node-01 -- GitLab From 26ca1e586aaf56d0d0de07b46723b016ecbdd11b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 11:51:10 +0000 Subject: [PATCH 54/82] QKD E2E Test: - Addec cleanup layer in GitLab CI/CD pipeline --- src/tests/qkd_end2end/.gitlab-ci.yml | 39 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 0b66487f0..800eb65d2 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -45,10 +45,43 @@ end2end_test qkd_end2end: #needs: # - build qkd_end2end before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + + # Always delete Kubernetes namespaces + - kubectl delete namespaces tfs || true + - kubectl delete namespaces crdb || true + - kubectl delete namespaces qdb || true + - kubectl delete namespaces kafka || true + - helm3 uninstall -n nats nats + - kubectl delete namespaces nats || true + + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f ${TEST_NAME} || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br script: # Download Docker image to run the test -- GitLab From ace2592311d733b76a4c488c11e9c4102d0bfd00 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 11:58:56 +0000 Subject: [PATCH 55/82] QKD E2E Test: - Generalized cleanup at beginning of GitLab CI/CD pipeline --- src/tests/qkd_end2end/.gitlab-ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 800eb65d2..ebe84b287 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -58,13 +58,6 @@ end2end_test qkd_end2end: - microk8s status --wait-ready # Always delete Kubernetes namespaces - - kubectl delete namespaces tfs || true - - kubectl delete namespaces crdb || true - - kubectl delete namespaces qdb || true - - kubectl delete namespaces kafka || true - - helm3 uninstall -n nats nats - - kubectl delete namespaces nats || true - - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" -- GitLab From ecb3e3da688ab4807aedf68a0c57b0f6e666b78f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 12:03:18 +0000 Subject: [PATCH 56/82] QKD E2E Test: - Fixed helm command in pre-cleanup of GitLab CI/CD pipeline --- src/tests/qkd_end2end/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index ebe84b287..bc10757dc 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -66,7 +66,7 @@ end2end_test qkd_end2end: - > for ns in ${OLD_NATS_NAMESPACES}; do if [[ "$ns" == nats* ]]; then - helm uninstall "$ns" -n "$ns" + helm3 uninstall "$ns" -n "$ns" fi done - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') -- GitLab From 4f8efa74893454e85b51330f3cb2ddddc6da7b2b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 12:08:39 +0000 Subject: [PATCH 57/82] Deployment: - Fixed Kafka namespace in initial checks + typos - Added check to see cockroachDB started --- deploy/crdb.sh | 7 +++++++ deploy/kafka.sh | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 0cff8594d..6e487d683 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -105,6 +105,13 @@ function crdb_deploy_single() { sleep 1 done kubectl wait --namespace ${CRDB_NAMESPACE} --for=condition=Ready --timeout=300s pod/cockroachdb-0 + + # Wait for CockroachDB to notify "start_node_query" + echo ">>> CockroachDB pods created. Waiting CockroachDB server to be started..." + while ! kubectl --namespace ${CRDB_NAMESPACE} logs pod/cockroachdb-0 -c kafka 2>&1 | grep -q 'start_node_query'; do + printf "%c" "." + sleep 1 + done fi echo diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 77300c216..a971c15d5 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -56,7 +56,7 @@ function kfk_deploy_single() { echo "Kafka (single-mode)" echo ">>> Checking if Kafka is deployed..." - if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/kafka &> /dev/null; then + if kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; then echo ">>> Kafka is present; skipping step." else echo ">>> Deploy Kafka" @@ -64,7 +64,7 @@ function kfk_deploy_single() { #sed -i "s//${KFK_NAMESPACE}/" "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" - echo ">>> Waiting Kakfa statefulset to be created..." + echo ">>> Waiting Kafka statefulset to be created..." while ! kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; do printf "%c" "." sleep 1 @@ -78,7 +78,7 @@ function kfk_deploy_single() { #kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/kafka #kubectl wait --namespace ${KGK_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \ # statefulset/kafka - echo ">>> Kakfa statefulset created. Waiting Kakfa pods to be created..." + echo ">>> Kafka statefulset created. Waiting Kafka pods to be created..." while ! kubectl get --namespace ${KFK_NAMESPACE} pod/kafka-0 &> /dev/null; do printf "%c" "." sleep 1 @@ -86,8 +86,8 @@ function kfk_deploy_single() { kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Ready --timeout=300s pod/kafka-0 # Wait for Kafka to notify "Kafka Server started" - echo ">>> Kakfa pods created. Waiting Kakfa Server be started..." - while ! kubectl --namespace $KFK_NAMESPACE logs pod/kafka-0 -c kafka 2>&1 | grep -q 'Kafka Server started'; do + echo ">>> Kafka pods created. Waiting Kafka Server to be started..." + while ! kubectl --namespace ${KFK_NAMESPACE} logs pod/kafka-0 -c kafka 2>&1 | grep -q 'Kafka Server started'; do printf "%c" "." sleep 1 done -- GitLab From e1aaca14e8719b379e4be1174b8a102a27ef111b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 13:12:08 +0000 Subject: [PATCH 58/82] QKD E2E Test: - Added missing dependency in Dockerfile --- src/tests/qkd_end2end/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/qkd_end2end/Dockerfile b/src/tests/qkd_end2end/Dockerfile index c260c90e4..4486522b6 100644 --- a/src/tests/qkd_end2end/Dockerfile +++ b/src/tests/qkd_end2end/Dockerfile @@ -67,6 +67,8 @@ COPY src/device/__init__.py device/__init__.py COPY src/device/client/. device/client/ COPY src/monitoring/__init__.py monitoring/__init__.py COPY src/monitoring/client/. monitoring/client/ +COPY src/qkd_app/__init__.py qkd_app/__init__.py +COPY src/qkd_app/client/. qkd_app/client/ COPY src/service/__init__.py service/__init__.py COPY src/service/client/. service/client/ COPY src/slice/__init__.py slice/__init__.py -- GitLab From dbd3e5e60605ddf87ea335a60068f55e438d2884 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 13:50:11 +0000 Subject: [PATCH 59/82] Deployment: - Fixed check to see cockroachDB started --- deploy/crdb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 6e487d683..4d6461940 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -108,7 +108,7 @@ function crdb_deploy_single() { # Wait for CockroachDB to notify "start_node_query" echo ">>> CockroachDB pods created. Waiting CockroachDB server to be started..." - while ! kubectl --namespace ${CRDB_NAMESPACE} logs pod/cockroachdb-0 -c kafka 2>&1 | grep -q 'start_node_query'; do + while ! kubectl --namespace ${CRDB_NAMESPACE} logs pod/cockroachdb-0 -c cockroachdb 2>&1 | grep -q 'start_node_query'; do printf "%c" "." sleep 1 done -- GitLab From 61b1a65cb13dbec3a620ecd0c8577c3615fa9d19 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 14:17:58 +0000 Subject: [PATCH 60/82] QKD E2E Test: - Corrected JSON data filenames - Addec cleanup of Failed system pods --- src/tests/qkd_end2end/.gitlab-ci.yml | 6 ++++++ src/tests/qkd_end2end/tests/test_01_onboarding.py | 2 +- src/tests/qkd_end2end/tests/test_02_create_links.py | 6 +++--- src/tests/qkd_end2end/tests/test_03_create_external_app.py | 2 +- src/tests/qkd_end2end/tests/test_06_cleanup.py | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index bc10757dc..45a9376a5 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -73,6 +73,12 @@ end2end_test qkd_end2end: - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" - kubectl delete namespace ${OLD_NAMESPACES} || true + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name + --no-headers | xargs -r -n2 kubectl delete pod -n + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY diff --git a/src/tests/qkd_end2end/tests/test_01_onboarding.py b/src/tests/qkd_end2end/tests/test_01_onboarding.py index 763d7da17..e609856e7 100644 --- a/src/tests/qkd_end2end/tests/test_01_onboarding.py +++ b/src/tests/qkd_end2end/tests/test_01_onboarding.py @@ -24,7 +24,7 @@ from .Fixtures import context_client, device_client # pylint: disable=unused-imp LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-topology.json') +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-01-topology.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_onboarding( diff --git a/src/tests/qkd_end2end/tests/test_02_create_links.py b/src/tests/qkd_end2end/tests/test_02_create_links.py index fb8a91342..0f6440380 100644 --- a/src/tests/qkd_end2end/tests/test_02_create_links.py +++ b/src/tests/qkd_end2end/tests/test_02_create_links.py @@ -60,7 +60,7 @@ def test_create_direct_link_qkd1_qkd2( device_client : DeviceClient, # pylint: disable=redefined-outer-name service_client : ServiceClient, # pylint: disable=redefined-outer-name ): - descriptor_file = compose_path('tfs-direct-link-qkd1-qkd2.json') + descriptor_file = compose_path('tfs-02-direct-link-qkd1-qkd2.json') # Load descriptors and validate the base scenario descriptor_loader = DescriptorLoader( @@ -106,7 +106,7 @@ def test_create_direct_link_qkd2_qkd3( device_client : DeviceClient, # pylint: disable=redefined-outer-name service_client : ServiceClient, # pylint: disable=redefined-outer-name ): - descriptor_file = compose_path('tfs-direct-link-qkd2-qkd3.json') + descriptor_file = compose_path('tfs-03-direct-link-qkd2-qkd3.json') # Load descriptors and validate the base scenario descriptor_loader = DescriptorLoader( @@ -152,7 +152,7 @@ def test_create_virtual_link_qkd1_qkd3( device_client : DeviceClient, # pylint: disable=redefined-outer-name service_client : ServiceClient, # pylint: disable=redefined-outer-name ): - descriptor_file = compose_path('tfs-virtual-link-qkd1-qkd3.json'), + descriptor_file = compose_path('tfs-04-virtual-link-qkd1-qkd3.json'), # Load descriptors and validate the base scenario descriptor_loader = DescriptorLoader( diff --git a/src/tests/qkd_end2end/tests/test_03_create_external_app.py b/src/tests/qkd_end2end/tests/test_03_create_external_app.py index fed2370ff..f9efd8aba 100644 --- a/src/tests/qkd_end2end/tests/test_03_create_external_app.py +++ b/src/tests/qkd_end2end/tests/test_03_create_external_app.py @@ -94,7 +94,7 @@ def test_create_external_app_qkd1_to_qkd3( def test_create_external_app_qkd3_to_qkd1( qkd_app_client : QKDAppClient, ): - request_file = compose_path('tfs-05-app-1-qkd3-qkd1.json') + request_file = compose_path('tfs-06-app-1-qkd3-qkd1.json') # Issue external QKD App creation request (QKD3-QKD1) with open(request_file, 'r', encoding='UTF-8') as f: diff --git a/src/tests/qkd_end2end/tests/test_06_cleanup.py b/src/tests/qkd_end2end/tests/test_06_cleanup.py index 20afb5fe0..e3c770e7a 100644 --- a/src/tests/qkd_end2end/tests/test_06_cleanup.py +++ b/src/tests/qkd_end2end/tests/test_06_cleanup.py @@ -24,7 +24,7 @@ from .Fixtures import context_client, device_client # pylint: disable=unused- LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) -DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-topology.json') +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-01-topology.json') ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) def test_scenario_cleanup( -- GitLab From ec4734f6280c6f4ec3bb2579f1f94d8200458399 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 14:21:34 +0000 Subject: [PATCH 61/82] QKD E2E Test: - Fixed cleanup of Failed system pods --- src/tests/qkd_end2end/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 45a9376a5..68a8780dd 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -75,9 +75,9 @@ end2end_test qkd_end2end: # Clean-up Kubernetes Failed pods - > - kubectl get pods --all-namespaces --field-selector=status.phase=Failed - -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name - --no-headers | xargs -r -n2 kubectl delete pod -n + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY -- GitLab From 6823a996d2970e8958fe0003c8ecff6ed82c17eb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 15:19:46 +0000 Subject: [PATCH 62/82] QKD E2E Test: - Fixed device name-uuid mapping - Fixed descriptor_file generation --- src/tests/qkd_end2end/tests/test_02_create_links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/qkd_end2end/tests/test_02_create_links.py b/src/tests/qkd_end2end/tests/test_02_create_links.py index 0f6440380..6a9cce80f 100644 --- a/src/tests/qkd_end2end/tests/test_02_create_links.py +++ b/src/tests/qkd_end2end/tests/test_02_create_links.py @@ -152,7 +152,7 @@ def test_create_virtual_link_qkd1_qkd3( device_client : DeviceClient, # pylint: disable=redefined-outer-name service_client : ServiceClient, # pylint: disable=redefined-outer-name ): - descriptor_file = compose_path('tfs-04-virtual-link-qkd1-qkd3.json'), + descriptor_file = compose_path('tfs-04-virtual-link-qkd1-qkd3.json') # Load descriptors and validate the base scenario descriptor_loader = DescriptorLoader( @@ -202,7 +202,7 @@ def test_check_qkd_apps_after( len(response.devices), grpc_message_to_json_string(response) )) device_uuid_to_name = { - device.device_id.device_uuid.uuid : device.name + device.name : device.device_id.device_uuid.uuid for device in response.devices } -- GitLab From 021782e49411afe9b92e10d1476a65e6104de56a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 15:20:40 +0000 Subject: [PATCH 63/82] QKD App: - Fixed generation of UUIDs - Fixed logic to handle get/delete/list apps --- .../service/QKDAppServiceServicerImpl.py | 42 ++---------- src/qkd_app/service/database/QKDApp.py | 65 ++++++++++--------- src/qkd_app/service/database/uuids/Context.py | 37 +++++++++++ src/qkd_app/service/database/uuids/QKDApp.py | 30 ++++++--- 4 files changed, 96 insertions(+), 78 deletions(-) create mode 100644 src/qkd_app/service/database/uuids/Context.py diff --git a/src/qkd_app/service/QKDAppServiceServicerImpl.py b/src/qkd_app/service/QKDAppServiceServicerImpl.py index 4179d4047..c58b0abaf 100644 --- a/src/qkd_app/service/QKDAppServiceServicerImpl.py +++ b/src/qkd_app/service/QKDAppServiceServicerImpl.py @@ -135,52 +135,22 @@ class AppServiceServicerImpl(AppServiceServicer): app_set(self.db_engine, self.messagebroker, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def ListApps(self, request: ContextId, context: grpc.ServicerContext) -> AppList: + def ListApps(self, request : ContextId, context : grpc.ServicerContext) -> AppList: """ Lists all apps in the system, including their statistics and QoS attributes. """ - LOGGER.debug(f"Received ListApps request: {grpc_message_to_json_string(request)}") - - try: - apps = app_list_objs(self.db_engine, request.context_uuid.uuid) - for app in apps.apps: - LOGGER.debug(f"App retrieved: {grpc_message_to_json_string(app)}") - - LOGGER.debug(f"ListApps returned {len(apps.apps)} apps for context_id: {request.context_uuid.uuid}") - return apps - except Exception as e: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details("An internal error occurred while listing apps.") - raise e + return app_list_objs(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetApp(self, request: AppId, context: grpc.ServicerContext) -> App: + def GetApp(self, request : AppId, context : grpc.ServicerContext) -> App: """ Fetches details of a specific app based on its AppId, including QoS and performance stats. """ - LOGGER.debug(f"Received GetApp request: {grpc_message_to_json_string(request)}") - try: - app = app_get(self.db_engine, request) - LOGGER.debug(f"GetApp found app with app_uuid: {request.app_uuid.uuid}") - return app - except NotFoundException as e: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details(f"App not found: {e}") - raise e + return app_get(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteApp(self, request: AppId, context: grpc.ServicerContext) -> Empty: + def DeleteApp(self, request : AppId, context : grpc.ServicerContext) -> Empty: """ Deletes an app from the system by its AppId, following ETSI compliance. """ - LOGGER.debug(f"Received DeleteApp request for app_uuid: {request.app_uuid.uuid}") - try: - app_delete(self.db_engine, request.app_uuid.uuid) - LOGGER.debug(f"App with UUID {request.app_uuid.uuid} deleted successfully.") - return Empty() - except NotFoundException as e: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details(f"App not found: {e}") - raise e - - + return app_delete(self.db_engine, request) diff --git a/src/qkd_app/service/database/QKDApp.py b/src/qkd_app/service/database/QKDApp.py index c724ef660..00d9cb7ac 100644 --- a/src/qkd_app/service/database/QKDApp.py +++ b/src/qkd_app/service/database/QKDApp.py @@ -23,9 +23,8 @@ from sqlalchemy_cockroachdb import run_transaction from common.method_wrappers.ServiceExceptions import NotFoundException from common.message_broker.MessageBroker import MessageBroker +from common.proto.context_pb2 import ContextId, Empty from common.proto.qkd_app_pb2 import AppList, App, AppId -from qkd_app.service.database.uuids._Builder import get_uuid_from_string, get_uuid_random -from common.method_wrappers.ServiceExceptions import InvalidArgumentsException from common.tools.object_factory.QKDApp import json_app_id from common.tools.object_factory.Context import json_context_id @@ -38,27 +37,25 @@ from .models.enums.QKDAppTypes import grpc_to_enum__qkd_app_types LOGGER = logging.getLogger(__name__) -def app_list_objs(db_engine: Engine, context_uuid: str = None) -> AppList: +def app_list_objs(db_engine : Engine, request : ContextId) -> AppList: """ Fetches a list of all QKD applications from the database. Optionally filters by context UUID. :param db_engine: SQLAlchemy Engine for DB connection - :param context_uuid: UUID of the context to filter by (optional) + :param request: Context Id containing the UUID of the context to filter by :return: AppList containing all apps """ - def callback(session: Session) -> List[Dict]: + context_uuid = context_get_uuid(request, allow_random=False) + def callback(session : Session) -> List[Dict]: query = session.query(AppModel) - - if context_uuid: - query = query.filter_by(context_uuid=context_uuid) - - return [obj.dump() for obj in query.all()] - + query = query.filter_by(context_uuid=context_uuid) + obj_list : List[AppModel] = query.all() + return [obj.dump() for obj in obj_list] apps = run_transaction(sessionmaker(bind=db_engine), callback) return AppList(apps=apps) -def app_get(db_engine: Engine, request: AppId) -> App: +def app_get(db_engine : Engine, request : AppId) -> App: """ Fetches a specific app by its UUID. @@ -67,23 +64,27 @@ def app_get(db_engine: Engine, request: AppId) -> App: :return: App protobuf object :raises NotFoundException: If the app is not found in the database """ - app_uuid = app_get_uuid(request, allow_random=False) + context_uuid,app_uuid = app_get_uuid(request, allow_random=False) - def callback(session: Session) -> Optional[Dict]: - obj = session.query(AppModel).filter_by(app_uuid=app_uuid).one_or_none() - return obj.dump() if obj else None + def callback(session : Session) -> Optional[Dict]: + query = session.query(AppModel) + query = query.filter_by(app_uuid=app_uuid) + obj : Optional[AppModel] = query.one_or_none() + return None if obj is None else obj.dump() obj = run_transaction(sessionmaker(bind=db_engine), callback) - - if not obj: - raise NotFoundException('App', request.app_uuid.uuid, extra_details=[ - f'app_uuid generated was: {app_uuid}' + + if obj is None: + raw_app_uuid = '{:s}/{:s}'.format(request.context_id.context_uuid.uuid, request.app_uuid.uuid) + raise NotFoundException('App', raw_app_uuid, extra_details=[ + 'context_uuid generated was: {:s}'.format(context_uuid), + 'app_uuid generated was: {:s}'.format(app_uuid), ]) - + return App(**obj) -def app_set(db_engine: Engine, messagebroker: MessageBroker, request: App) -> AppId: +def app_set(db_engine : Engine, messagebroker : MessageBroker, request : App) -> AppId: """ Creates or updates an app in the database. If the app already exists, updates the app. Otherwise, inserts a new entry. @@ -93,8 +94,7 @@ def app_set(db_engine: Engine, messagebroker: MessageBroker, request: App) -> Ap :param request: App protobuf object containing app data :return: AppId protobuf object representing the newly created or updated app """ - context_uuid = context_get_uuid(request.app_id.context_id, allow_random=False) - app_uuid = app_get_uuid(request.app_id, allow_random=True) + context_uuid,app_uuid = app_get_uuid(request.app_id, allow_random=True) # Prepare app data for insertion/update app_data = { @@ -154,20 +154,21 @@ def app_get_by_server(db_engine: Engine, server_app_id: str) -> App: return App(**obj) -def app_delete(db_engine: Engine, app_uuid: str) -> None: +def app_delete(db_engine : Engine, request : AppId) -> Empty: """ Deletes an app by its UUID from the database. :param db_engine: SQLAlchemy Engine for DB connection - :param app_uuid: The UUID of the app to be deleted + :param app_id: The UUID of the app to be deleted """ - def callback(session: Session) -> bool: - app_obj = session.query(AppModel).filter_by(app_uuid=app_uuid).one_or_none() - if app_obj is None: - raise NotFoundException('App', app_uuid) + _,app_uuid = app_get_uuid(request, allow_random=False) - session.delete(app_obj) - return True + def callback(session : Session) -> bool: + query = session.query(AppModel) + query = query.filter_by(app_uuid=app_uuid) + num_deleted = query.delete() + return num_deleted > 0 run_transaction(sessionmaker(bind=db_engine), callback) + return Empty() diff --git a/src/qkd_app/service/database/uuids/Context.py b/src/qkd_app/service/database/uuids/Context.py new file mode 100644 index 000000000..752e62efe --- /dev/null +++ b/src/qkd_app/service/database/uuids/Context.py @@ -0,0 +1,37 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.method_wrappers.ServiceExceptions import InvalidArgumentsException +from ._Builder import get_uuid_from_string, get_uuid_random + +def context_get_uuid( + context_id : ContextId, context_name : str = '', allow_random : bool = False, allow_default : bool = False +) -> str: + context_uuid = context_id.context_uuid.uuid + + if len(context_uuid) > 0: + return get_uuid_from_string(context_uuid) + if len(context_name) > 0: + return get_uuid_from_string(context_name) + if allow_default: + return get_uuid_from_string(DEFAULT_CONTEXT_NAME) + if allow_random: + return get_uuid_random() + + raise InvalidArgumentsException([ + ('context_id.context_uuid.uuid', context_uuid), + ('name', context_name), + ], extra_details=['At least one is required to produce a Context UUID']) diff --git a/src/qkd_app/service/database/uuids/QKDApp.py b/src/qkd_app/service/database/uuids/QKDApp.py index 86f33f58e..135ac9ec6 100644 --- a/src/qkd_app/service/database/uuids/QKDApp.py +++ b/src/qkd_app/service/database/uuids/QKDApp.py @@ -12,26 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Tuple from common.proto.qkd_app_pb2 import AppId from common.method_wrappers.ServiceExceptions import InvalidArgumentsException from ._Builder import get_uuid_from_string, get_uuid_random +from .Context import context_get_uuid -def app_get_uuid(app_id: AppId, allow_random: bool = False) -> str: +def app_get_uuid( + app_id : AppId, app_name : str = '', allow_random : bool = False +) -> Tuple[str, str]: """ Retrieves or generates the UUID for an app. - :param app_id: AppId object that contains the app UUID + :param application_id: AppId object that contains the app UUID + :param application_name: string that contains optional app name :param allow_random: If True, generates a random UUID if app_uuid is not set - :return: App UUID as a string + :return: Context UUID as a string , App UUID as a string """ - app_uuid = app_id.app_uuid.uuid + context_uuid = context_get_uuid(app_id.context_id, allow_random=False, allow_default=True) + raw_app_uuid = app_id.app_uuid.uuid + + if len(raw_app_uuid) > 0: + return context_uuid, get_uuid_from_string(raw_app_uuid, prefix_for_name=context_uuid) + + if len(app_name) > 0: + return context_uuid, get_uuid_from_string(app_name, prefix_for_name=context_uuid) - if app_uuid: - return get_uuid_from_string(app_uuid) - if allow_random: - return get_uuid_random() + return context_uuid, get_uuid_random() raise InvalidArgumentsException([ - ('app_id.app_uuid.uuid', app_uuid), - ], extra_details=['At least one UUID is required to identify the app.']) + ('app_id.app_uuid.uuid', raw_app_uuid), + ('name', app_name), + ], extra_details=['At least one is required to produce a App UUID']) -- GitLab From bd182726fbbaeba2530df76a95e4b621a6cfe5af Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 15:58:44 +0000 Subject: [PATCH 64/82] QKD E2E Test: - Fixed create links test --- src/tests/qkd_end2end/tests/test_02_create_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/qkd_end2end/tests/test_02_create_links.py b/src/tests/qkd_end2end/tests/test_02_create_links.py index 6a9cce80f..4d7587cf3 100644 --- a/src/tests/qkd_end2end/tests/test_02_create_links.py +++ b/src/tests/qkd_end2end/tests/test_02_create_links.py @@ -230,6 +230,6 @@ def test_check_qkd_apps_after( remote_device_id = app.remote_device_id.device_uuid.uuid device_pair = (local_device_id, remote_device_id) assert device_pair in pending_device_pairs - pending_device_pairs.pop(device_pair) + pending_device_pairs.remove(device_pair) assert len(pending_device_pairs) == 0 -- GitLab From e2007cbc7ea323cc45aaf8f5ea6d59938556d162 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 15:59:04 +0000 Subject: [PATCH 65/82] GitLab CI/CD pipeline: - Disabled unneeded test --- src/tests/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index 5a0dd6883..e1ac5e9b3 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -27,5 +27,5 @@ include: # #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' - local: '/src/tests/qkd_end2end/.gitlab-ci.yml' - - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' +# - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' - local: '/src/tests/tools/mock_qkd_node/.gitlab-ci.yml' -- GitLab From b12e2fa97ba57dadb99ee0b75389eaedd17ee9ff Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 16:37:54 +0000 Subject: [PATCH 66/82] QKD E2E Test: - Added wait loop for kubernetes to be ready --- src/tests/qkd_end2end/.gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 68a8780dd..329a9c062 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -137,6 +137,18 @@ end2end_test qkd_end2end: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment -- GitLab From 4f2931c945455f6704a83af956689132b2430147 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 16:54:29 +0000 Subject: [PATCH 67/82] QKD E2E Test: - Fixed check of remaining qkd apps during delete --- src/tests/qkd_end2end/tests/test_04_delete_external_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/qkd_end2end/tests/test_04_delete_external_app.py b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py index 8c8236e6e..f352498a8 100644 --- a/src/tests/qkd_end2end/tests/test_04_delete_external_app.py +++ b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py @@ -104,7 +104,7 @@ def test_check_qkd_apps_after( LOGGER.warning('QKDApps[{:d}] = {:s}'.format( len(response.apps), grpc_message_to_json_string(response) )) - assert len(response.apps) == 3 + assert len(response.apps) == 2 num_internal = 0 num_external = 0 -- GitLab From 0d877c0a3a1887ad397e1a7437de89da4defff8f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 16:55:46 +0000 Subject: [PATCH 68/82] GitLab CI/CD pipeline: - Upgraded other CI/CD tests to meet new MicroK8s/Docker checks/cleanup - Reactivated those tests --- src/tests/.gitlab-ci.yml | 24 ++++++------ src/tests/ecoc22/.gitlab-ci.yml | 65 +++++++++++++++++++++++++++++++- src/tests/eucnc24/.gitlab-ci.yml | 62 +++++++++++++++++++++++++++--- src/tests/ofc22/.gitlab-ci.yml | 64 ++++++++++++++++++++++++++++++- src/tests/ofc24/.gitlab-ci.yml | 63 +++++++++++++++++++++++++++---- 5 files changed, 249 insertions(+), 29 deletions(-) diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index e1ac5e9b3..4b7af45a0 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -14,18 +14,18 @@ # include the individual .gitlab-ci.yml of each end-to-end integration test include: -# - local: '/src/tests/ofc22/.gitlab-ci.yml' -# #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' -# - local: '/src/tests/ecoc22/.gitlab-ci.yml' -# #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' -# #- local: '/src/tests/ofc23/.gitlab-ci.yml' -# - local: '/src/tests/ofc24/.gitlab-ci.yml' -# - local: '/src/tests/eucnc24/.gitlab-ci.yml' -# #- local: '/src/tests/ofc25-camara-agg-net-controller/.gitlab-ci.yml' -# #- local: '/src/tests/ofc25-camara-e2e-controller/.gitlab-ci.yml' -# #- local: '/src/tests/ofc25/.gitlab-ci.yml' -# #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' + - local: '/src/tests/ofc22/.gitlab-ci.yml' + #- local: '/src/tests/oeccpsc22/.gitlab-ci.yml' + - local: '/src/tests/ecoc22/.gitlab-ci.yml' + #- local: '/src/tests/nfvsdn22/.gitlab-ci.yml' + #- local: '/src/tests/ofc23/.gitlab-ci.yml' + - local: '/src/tests/ofc24/.gitlab-ci.yml' + - local: '/src/tests/eucnc24/.gitlab-ci.yml' + #- local: '/src/tests/ofc25-camara-agg-net-controller/.gitlab-ci.yml' + #- local: '/src/tests/ofc25-camara-e2e-controller/.gitlab-ci.yml' + #- local: '/src/tests/ofc25/.gitlab-ci.yml' + #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' - local: '/src/tests/qkd_end2end/.gitlab-ci.yml' -# - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' + - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' - local: '/src/tests/tools/mock_qkd_node/.gitlab-ci.yml' diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index aaab9f4f4..bf824c985 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -45,13 +45,61 @@ end2end_test ecoc22: #needs: # - build ecoc22 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment @@ -62,6 +110,7 @@ end2end_test ecoc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -76,7 +125,7 @@ end2end_test ecoc22: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -88,7 +137,9 @@ end2end_test ecoc22: --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server @@ -96,8 +147,18 @@ end2end_test ecoc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Clean up + - docker rm -f ${TEST_NAME} || true + - kubectl delete namespaces tfs || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br + - docker volume prune --force + - docker image prune --force + + # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/tests/eucnc24/.gitlab-ci.yml b/src/tests/eucnc24/.gitlab-ci.yml index e63e0521b..960151473 100644 --- a/src/tests/eucnc24/.gitlab-ci.yml +++ b/src/tests/eucnc24/.gitlab-ci.yml @@ -45,16 +45,64 @@ end2end_test eucnc24: #needs: # - build eucnc24 before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f ${TEST_NAME} || true + # Cleanup old ContainerLab scenarios - containerlab destroy --all --cleanup || true + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Deploy ContainerLab Scenario @@ -83,7 +131,6 @@ end2end_test eucnc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" @@ -95,7 +142,7 @@ end2end_test eucnc24: - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/kafka.sh - - ./deploy/qdb.sh + #- ./deploy/qdb.sh - ./deploy/tfs.sh - ./deploy/show.sh @@ -233,9 +280,8 @@ end2end_test eucnc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - # Destroy Scenario + # Clean up - docker rm -f ${TEST_NAME} || true - RUNNER_PATH=`pwd` #- cd $PWD/src/tests/${TEST_NAME} @@ -244,6 +290,10 @@ end2end_test eucnc24: - sudo rm -rf clab-eucnc24/ .eucnc24.clab.yml.bak || true - cd $RUNNER_PATH - kubectl delete namespaces tfs || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br + - docker volume prune --force + - docker image prune --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 90adc40e9..90d6a7b86 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -45,13 +45,61 @@ end2end_test ofc22: #needs: # - build ofc22 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment @@ -63,6 +111,7 @@ end2end_test ofc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -77,7 +126,7 @@ end2end_test ofc22: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -90,6 +139,7 @@ end2end_test ofc22: --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server @@ -99,8 +149,18 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c ztpservice - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Clean up + - docker rm -f ${TEST_NAME} || true + - kubectl delete namespaces tfs || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br + - docker volume prune --force + - docker image prune --force + + # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml index 0a31dd21f..1bbeef5b5 100644 --- a/src/tests/ofc24/.gitlab-ci.yml +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -45,9 +45,42 @@ end2end_test ofc24: #needs: # - build ofc24 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f na-t1 na-t2 na-r1 na-r2 - - docker network rm -f na-br script: # Download Docker image to run the test @@ -57,6 +90,18 @@ end2end_test ofc24: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Deploy Optical Device Node Agents @@ -104,6 +149,7 @@ end2end_test ofc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -118,7 +164,7 @@ end2end_test ofc24: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -140,7 +186,6 @@ end2end_test ofc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi # Dump Optical Device Node Agents container status and logs - docker ps -a @@ -149,9 +194,13 @@ end2end_test ofc24: - docker logs na-r1 - docker logs na-r2 - # Destroy Optical Device Node Agents - - docker rm -f na-t1 na-t2 na-r1 na-r2 - - docker network rm -f na-br + # Clean up + - docker rm -f ${TEST_NAME} || true + - kubectl delete namespaces tfs || true + - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 + - docker network rm --force qkd-node-br + - docker volume prune --force + - docker image prune --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi -- GitLab From f016365407085090de59b3953789417545fe6ca7 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 17:16:37 +0000 Subject: [PATCH 69/82] GitLab CI/CD pipeline: - Upgraded CI/CD end-to-end tests to properly cleanup - --- src/tests/ecoc22/.gitlab-ci.yml | 12 ++++++------ src/tests/eucnc24/.gitlab-ci.yml | 10 +++++----- src/tests/ofc22/.gitlab-ci.yml | 13 +++++++------ src/tests/ofc24/.gitlab-ci.yml | 26 +++++++++++++------------- src/tests/qkd_end2end/.gitlab-ci.yml | 11 ++++++----- 5 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index bf824c985..82e004b52 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -133,7 +133,7 @@ end2end_test ecoc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest @@ -149,12 +149,12 @@ end2end_test ecoc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server # Clean up - - docker rm -f ${TEST_NAME} || true - kubectl delete namespaces tfs || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br - - docker volume prune --force - - docker image prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/eucnc24/.gitlab-ci.yml b/src/tests/eucnc24/.gitlab-ci.yml index 960151473..9d47232c9 100644 --- a/src/tests/eucnc24/.gitlab-ci.yml +++ b/src/tests/eucnc24/.gitlab-ci.yml @@ -282,7 +282,6 @@ end2end_test eucnc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server # Clean up - - docker rm -f ${TEST_NAME} || true - RUNNER_PATH=`pwd` #- cd $PWD/src/tests/${TEST_NAME} - cd /tmp/clab/${TEST_NAME} @@ -290,10 +289,11 @@ end2end_test eucnc24: - sudo rm -rf clab-eucnc24/ .eucnc24.clab.yml.bak || true - cd $RUNNER_PATH - kubectl delete namespaces tfs || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br - - docker volume prune --force - - docker image prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 90d6a7b86..62a8a8485 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -134,10 +134,11 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh @@ -151,12 +152,12 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c ztpservice # Clean up - - docker rm -f ${TEST_NAME} || true - kubectl delete namespaces tfs || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br - - docker volume prune --force - - docker image prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml index 1bbeef5b5..87de41b9a 100644 --- a/src/tests/ofc24/.gitlab-ci.yml +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -172,12 +172,19 @@ end2end_test ofc24: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: + # Dump Optical Device Node Agents container status and logs + - docker ps -a + - docker logs na-t1 + - docker logs na-t2 + - docker logs na-r1 + - docker logs na-r2 + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -187,20 +194,13 @@ end2end_test ofc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server - # Dump Optical Device Node Agents container status and logs - - docker ps -a - - docker logs na-t1 - - docker logs na-t2 - - docker logs na-r1 - - docker logs na-r2 - # Clean up - - docker rm -f ${TEST_NAME} || true - kubectl delete namespaces tfs || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br - - docker volume prune --force - - docker image prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index 329a9c062..f51ce0a6a 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -289,6 +289,7 @@ end2end_test qkd_end2end: after_script: # Dump logs of the QKD Nodes (after_script) + - docker ps -a - docker logs qkd-node-01 - docker logs qkd-node-02 - docker logs qkd-node-03 @@ -314,12 +315,12 @@ end2end_test qkd_end2end: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qkd-appservice -c server # Clean up - - docker rm -f ${TEST_NAME} || true - kubectl delete namespaces tfs || true - - docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 - - docker network rm --force qkd-node-br - - docker volume prune --force - - docker image prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi -- GitLab From 3436a7f8dc835271b23c107821cc2117104bc6c9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 17:18:18 +0000 Subject: [PATCH 70/82] QKD E2E Test: - Fixed wrong test filename --- src/tests/qkd_end2end/scripts/run-05-delete-links.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/qkd_end2end/scripts/run-05-delete-links.sh b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh index f00a8711f..8975759ce 100755 --- a/src/tests/qkd_end2end/scripts/run-05-delete-links.sh +++ b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh @@ -17,4 +17,4 @@ source /var/teraflow/tfs_runtime_env_vars.sh export PYTHONPATH=/var/teraflow pytest --verbose --log-level=INFO \ --junitxml=/opt/results/report_service_tfs_remove.xml \ - /var/teraflow/tests/qkd_end2end/tests/test_05-delete-links.py + /var/teraflow/tests/qkd_end2end/tests/test_05_delete_links.py -- GitLab From 75be39034f86f64b698dc211cd5864e6c29f0c21 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 17:50:41 +0000 Subject: [PATCH 71/82] GitLab CI/CD pipeline: - Upgraded CI/CD end-to-end tests to wait for MicroK8s to be ready - Expanded test timeout to 45 minutes --- src/tests/ecoc22/.gitlab-ci.yml | 15 ++++++++++++++- src/tests/eucnc24/.gitlab-ci.yml | 15 ++++++++++++++- src/tests/ofc22/.gitlab-ci.yml | 15 ++++++++++++++- src/tests/ofc24/.gitlab-ci.yml | 15 ++++++++++++++- src/tests/qkd_end2end/.gitlab-ci.yml | 15 ++++++++++++++- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 82e004b52..968b2333a 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ecoc22: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ecoc22: - timeout: 20m + timeout: 45m variables: TEST_NAME: 'ecoc22' stage: end2end_test @@ -56,6 +56,19 @@ end2end_test ecoc22: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces # Always delete Kubernetes namespaces - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') diff --git a/src/tests/eucnc24/.gitlab-ci.yml b/src/tests/eucnc24/.gitlab-ci.yml index 9d47232c9..70bdeba39 100644 --- a/src/tests/eucnc24/.gitlab-ci.yml +++ b/src/tests/eucnc24/.gitlab-ci.yml @@ -37,7 +37,7 @@ build eucnc24: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test eucnc24: - timeout: 90m + timeout: 45m variables: TEST_NAME: 'eucnc24' stage: end2end_test @@ -59,6 +59,19 @@ end2end_test eucnc24: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces # Always delete Kubernetes namespaces - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 62a8a8485..70f13f7d5 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ofc22: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ofc22: - timeout: 20m + timeout: 45m variables: TEST_NAME: 'ofc22' stage: end2end_test @@ -56,6 +56,19 @@ end2end_test ofc22: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces # Always delete Kubernetes namespaces - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml index 87de41b9a..6579d7c2e 100644 --- a/src/tests/ofc24/.gitlab-ci.yml +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ofc24: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ofc24: - timeout: 90m + timeout: 45m variables: TEST_NAME: 'ofc24' stage: end2end_test @@ -56,6 +56,19 @@ end2end_test ofc24: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces # Always delete Kubernetes namespaces - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml index f51ce0a6a..62d97f8ca 100644 --- a/src/tests/qkd_end2end/.gitlab-ci.yml +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -37,7 +37,7 @@ build qkd_end2end: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test qkd_end2end: - timeout: 30m + timeout: 45m variables: TEST_NAME: 'qkd_end2end' stage: end2end_test @@ -56,6 +56,19 @@ end2end_test qkd_end2end: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces # Always delete Kubernetes namespaces - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') -- GitLab From 8bc264ad54b7b14f303e7a435e16879fc8b557ba Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 17:53:00 +0000 Subject: [PATCH 72/82] QKD E2E Test: - Fixed delete virtual/direct links --- src/tests/qkd_end2end/tests/test_05_delete_links.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/tests/qkd_end2end/tests/test_05_delete_links.py b/src/tests/qkd_end2end/tests/test_05_delete_links.py index b04236d47..07a85963d 100644 --- a/src/tests/qkd_end2end/tests/test_05_delete_links.py +++ b/src/tests/qkd_end2end/tests/test_05_delete_links.py @@ -78,7 +78,7 @@ def test_delete_internal_apps( # Identify internal QKD apps to delete internal_app_uuids : Set[str] = set() for app in response.apps: - if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_CLIENT: continue + if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: continue internal_app_uuids.add(app.app_id.app_uuid.uuid) assert len(internal_app_uuids) == 2 @@ -119,13 +119,20 @@ def test_delete_services_associated_qkd_apps( assert len(virtual_link_service_uuids) == 1 assert len(direct_link_service_uuids ) == 2 - # Delete the services + # Delete the services for virtual links for svc_uuid in virtual_link_service_uuids: svc_id = ServiceId() svc_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME svc_id.service_uuid.uuid = svc_uuid response = service_client.DeleteService(svc_id) + # Delete the services for direct links + for svc_uuid in direct_link_service_uuids: + svc_id = ServiceId() + svc_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + svc_id.service_uuid.uuid = svc_uuid + response = service_client.DeleteService(svc_id) + # Check there are no services response = context_client.ListServices(ADMIN_CONTEXT_ID) LOGGER.warning('Services[{:d}] = {:s}'.format( -- GitLab From 7b132ba21ba099dbfa9dc1335b0061d7c3288856 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:16:29 +0000 Subject: [PATCH 73/82] PRe-merge cleanup --- manifests/contextservice.yaml | 2 +- manifests/deviceservice.yaml | 2 +- manifests/nbiservice.yaml | 4 ++-- manifests/pathcompservice.yaml | 2 +- manifests/qkd_appservice.yaml | 2 +- manifests/serviceservice.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 2bf8b5d86..5592864d6 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -40,7 +40,7 @@ spec: - name: MB_BACKEND value: "nats" - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY value: "FALSE" - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index 7c3ded7c0..a366a5041 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:2020"] diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index 6a07478c0..27026cc0f 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -39,9 +39,9 @@ spec: #- containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: FLASK_ENV - value: "development" # change to "development" if developing + value: "production" # normal value is "production", change to "development" if developing - name: IETF_NETWORK_RENDERER value: "LIBYANG" envFrom: diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 71c7e4cd7..2db0d41b0 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: ENABLE_FORECASTER value: "NO" readinessProbe: diff --git a/manifests/qkd_appservice.yaml b/manifests/qkd_appservice.yaml index 7641bd3aa..79cfaf6bd 100644 --- a/manifests/qkd_appservice.yaml +++ b/manifests/qkd_appservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: CRDB_DATABASE value: "tfs_qkd_app" envFrom: diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 8615e8879..8262550ef 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:3030"] -- GitLab From 9838a806b51bb62c76607d11f822b82172df3daf Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:23:22 +0000 Subject: [PATCH 74/82] Pre-merge cleanup --- src/tests/qkd_end2end/dump_logs.sh | 14 ++++++++++++++ src/tests/tools/mock_qkd_node/do_tests.sh | 14 ++++++++++++++ src/tests/tools/mock_qkd_node/run.sh | 2 +- src/tests/tools/mock_qkd_node/tests.sh | 14 ++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/tests/qkd_end2end/dump_logs.sh b/src/tests/qkd_end2end/dump_logs.sh index bde1f4720..9abc47d73 100755 --- a/src/tests/qkd_end2end/dump_logs.sh +++ b/src/tests/qkd_end2end/dump_logs.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + kubectl --namespace tfs logs deployment/contextservice -c server > context.log kubectl --namespace tfs logs deployment/deviceservice -c server > device.log diff --git a/src/tests/tools/mock_qkd_node/do_tests.sh b/src/tests/tools/mock_qkd_node/do_tests.sh index 9ace3467b..02df6dd34 100755 --- a/src/tests/tools/mock_qkd_node/do_tests.sh +++ b/src/tests/tools/mock_qkd_node/do_tests.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build the new Docker image echo "Building..." diff --git a/src/tests/tools/mock_qkd_node/run.sh b/src/tests/tools/mock_qkd_node/run.sh index 900f0994f..a017f9fed 100755 --- a/src/tests/tools/mock_qkd_node/run.sh +++ b/src/tests/tools/mock_qkd_node/run.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/tests/tools/mock_qkd_node/tests.sh b/src/tests/tools/mock_qkd_node/tests.sh index ea19eaa91..81febc4ec 100755 --- a/src/tests/tools/mock_qkd_node/tests.sh +++ b/src/tests/tools/mock_qkd_node/tests.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + echo "[QKD-NODE-01] Reading data on startup..." curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: -- GitLab From 9aca4539c48d3c81db301c0098da2cb1ff826f7a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:27:24 +0000 Subject: [PATCH 75/82] Pre-merge cleanup --- my_deploy.sh | 10 ++++----- src/tests/qkd_end2end/deploy_specs.sh | 24 ++++++++++----------- src/tests/qkd_end2end/redeploy-all.sh | 3 ++- src/tests/qkd_end2end/redeploy-qkd-nodes.sh | 3 ++- src/tests/qkd_end2end/redeploy-tfs.sh | 1 + src/tests/tools/mock_qkd_node/qkd_node.py | 15 +++++++++++++ 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index d3384276d..662dc389b 100644 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -80,11 +80,11 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # To manage QKD Apps, "service" requires "qkd_app" to be deployed # before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the # "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. -if [[ "$TFS_COMPONENTS" == *"service"* ]]; then - BEFORE="${TFS_COMPONENTS% service*}" - AFTER="${TFS_COMPONENTS#* service}" - export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" -fi +#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then +# BEFORE="${TFS_COMPONENTS% service*}" +# AFTER="${TFS_COMPONENTS#* service}" +# export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +#fi # Uncomment to activate Load Generator #export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" diff --git a/src/tests/qkd_end2end/deploy_specs.sh b/src/tests/qkd_end2end/deploy_specs.sh index 9e06cd7b4..9a487d83a 100755 --- a/src/tests/qkd_end2end/deploy_specs.sh +++ b/src/tests/qkd_end2end/deploy_specs.sh @@ -159,6 +159,18 @@ export NATS_DEPLOY_MODE="single" export NATS_REDEPLOY="" +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + +# Set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT="9092" + +# Set the flag to YES for redeploying of Apache Kafka +export KFK_REDEPLOY="" + + # ----- QuestDB ---------------------------------------------------------------- # Set the namespace where QuestDB will be deployed. @@ -199,15 +211,3 @@ export PROM_EXT_PORT_HTTP="9090" # Set the external port Grafana HTTP Dashboards will be exposed to. export GRAF_EXT_PORT_HTTP="3000" - - -# ----- Apache Kafka ----------------------------------------------------------- - -# Set the namespace where Apache Kafka will be deployed. -export KFK_NAMESPACE="kafka" - -# Set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT="9092" - -# Set the flag to YES for redeploying of Apache Kafka -export KFK_REDEPLOY="" diff --git a/src/tests/qkd_end2end/redeploy-all.sh b/src/tests/qkd_end2end/redeploy-all.sh index a1182de23..b534e2465 100755 --- a/src/tests/qkd_end2end/redeploy-all.sh +++ b/src/tests/qkd_end2end/redeploy-all.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + # Cleanup docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 docker network rm --force qkd-node-br diff --git a/src/tests/qkd_end2end/redeploy-qkd-nodes.sh b/src/tests/qkd_end2end/redeploy-qkd-nodes.sh index d4dc25752..b01e0ecb8 100755 --- a/src/tests/qkd_end2end/redeploy-qkd-nodes.sh +++ b/src/tests/qkd_end2end/redeploy-qkd-nodes.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + # Cleanup docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 docker network rm --force qkd-node-br diff --git a/src/tests/qkd_end2end/redeploy-tfs.sh b/src/tests/qkd_end2end/redeploy-tfs.sh index 7433ae4a1..81b0ac597 100755 --- a/src/tests/qkd_end2end/redeploy-tfs.sh +++ b/src/tests/qkd_end2end/redeploy-tfs.sh @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. + source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh ./deploy/all.sh diff --git a/src/tests/tools/mock_qkd_node/qkd_node.py b/src/tests/tools/mock_qkd_node/qkd_node.py index 9834a259b..e507a4619 100644 --- a/src/tests/tools/mock_qkd_node/qkd_node.py +++ b/src/tests/tools/mock_qkd_node/qkd_node.py @@ -1,3 +1,18 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from freeconf import restconf, source, device, parser, node, source, nodeutil from threading import Event -- GitLab From 6f1aa9b05cf78e2e52f4dfab88a61efad14ee2ad Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:30:38 +0000 Subject: [PATCH 76/82] Pre-merge cleanup --- src/service/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 6be7139ba..49efe9829 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -32,10 +32,6 @@ RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel RUN python3 -m pip install --upgrade pip-tools -# Install Flask globally -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install flask - # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow -- GitLab From 0cadde8a2d822c33d089e3eae7ee99405f06d86a Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:39:58 +0000 Subject: [PATCH 77/82] Pre-merge cleanup --- src/device/.gitlab-ci.yml | 207 ++++++++++++++++++++------------------ 1 file changed, 109 insertions(+), 98 deletions(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 6de43d37a..2db58fd13 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -12,16 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -stages: - - build - - prepare - - unit_test - # Build, tag, and push the Docker image to the GitLab Docker registry -build_device: +build device: variables: - IMAGE_NAME: 'device' - IMAGE_TAG: 'latest' + IMAGE_NAME: 'device' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: build before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY @@ -37,135 +32,151 @@ build_device: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - -# Deploy mock QKD nodes -prepare_mock_qkd_nodes: - stage: prepare - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge network is already created"; else docker network create --driver=bridge teraflowbridge; fi - - | - # Context-related cleanup - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml - script: - - docker volume create crdb - - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node - - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 - - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - MOCK_NODES_DIR="$CI_PROJECT_DIR/controller/src/tests/tools/mock_qkd_nodes" - - | - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" && ./start.sh & - MOCK_NODES_PID=$! - else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; - fi - - echo "Waiting for mock nodes to be up..." - - RETRY_COUNT=0 - - MAX_RETRIES=15 - - | - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - if curl -s http://127.0.0.1:11111 > /dev/null && \ - curl -s http://127.0.0.1:22222 > /dev/null && \ - curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!" - break - else - echo "Mock nodes not ready, retrying in 5 seconds..." - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 5 - fi - done - - | - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts." - exit 1 - fi - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/common/**/*.py - - proto/*.proto - - src/device/**/*.{py,in,yml} - - src/device/Dockerfile - - src/device/tests/*.py - - src/tests/tools/mock_qkd_nodes/** - - .gitlab-ci.yml +## Deploy mock QKD nodes +#prepare_mock_qkd_nodes: +# stage: prepare +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge network is already created"; else docker network create --driver=bridge teraflowbridge; fi +# - | +# # Context-related cleanup +# if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi +# if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi +# if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi +# +# script: +# - docker volume create crdb +# - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node +# - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 +# - echo "Waiting for initialization..." +# - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done +# - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done +# - MOCK_NODES_DIR="$CI_PROJECT_DIR/controller/src/tests/tools/mock_qkd_nodes" +# - | +# if [ -d "$MOCK_NODES_DIR" ]; then +# cd "$MOCK_NODES_DIR" && ./start.sh & +# MOCK_NODES_PID=$! +# else +# echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; +# fi +# - echo "Waiting for mock nodes to be up..." +# - RETRY_COUNT=0 +# - MAX_RETRIES=15 +# - | +# while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do +# if curl -s http://127.0.0.1:11111 > /dev/null && \ +# curl -s http://127.0.0.1:22222 > /dev/null && \ +# curl -s http://127.0.0.1:33333 > /dev/null; then +# echo "Mock nodes are up!" +# break +# else +# echo "Mock nodes not ready, retrying in 5 seconds..." +# RETRY_COUNT=$((RETRY_COUNT + 1)) +# sleep 5 +# fi +# done +# - | +# if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then +# echo "Error: Mock nodes failed to start after multiple attempts." +# exit 1 +# fi +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/device/**/*.{py,in,yml} +# - src/device/Dockerfile +# - src/device/tests/*.py +# - src/tests/tools/mock_qkd_nodes/** +# - .gitlab-ci.yml # Apply unit test to the component -unit_test_device: +unit_test device: variables: - IMAGE_NAME: 'device' - IMAGE_TAG: 'latest' + IMAGE_NAME: 'device' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: - - build_device - - prepare_mock_qkd_nodes + - build device before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - > + if docker network list | grep teraflowbridge; then + echo "teraflowbridge is already created"; + else + docker network create -d bridge teraflowbridge; + fi + - > + if docker container ls | grep $IMAGE_NAME; then + docker rm -f $IMAGE_NAME; + else + echo "$IMAGE_NAME image is not in the system"; + fi script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME --network=teraflowbridge -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" -e PYTHONPATH="/var/teraflow:/var/teraflow/device:/var/teraflow/tests/tools/mock_qkd_nodes:/var/teraflow/tests" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run --name $IMAGE_NAME -d -p 2020:2020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_error_handling.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_error_handling.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME - - docker rm -f pathcomp-frontend pathcomp-backend device context crdb nats - - docker volume rm -f crdb - docker network rm teraflowbridge - - docker volume prune --force - - docker image prune --force rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - src/tests/tools/mock_qkd_nodes/** - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - src/tests/tools/mock_qkd_nodes/** + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml artifacts: when: always reports: junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml ## Deployment of the service in Kubernetes Cluster -#deploy_device: +#deploy device: # variables: # IMAGE_NAME: 'device' # name of the microservice # IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) # stage: deploy # needs: -# - unit_test_device +# - unit test_device +# # - integ_test execute # script: # - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' # - kubectl version # - kubectl get all # - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" # - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test # rules: # - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' # when: manual -- GitLab From de963e742a73cff0d8137da08c1bf2d4d7fd10c2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 17 Jul 2025 18:49:11 +0000 Subject: [PATCH 78/82] Pre-merge cleanup --- .gitlab-ci.yml | 1 - src/service/.gitlab-ci.yml | 59 +++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edc86711e..c5d2bc451 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,6 @@ stages: #- dependencies - build - - prepare - unit_test - end2end_test diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 37c5bb0dc..1ac2326b8 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -49,22 +49,22 @@ unit_test service: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - | - # Context-related cleanup - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - # Device-related cleanup - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi + # Context-related + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi + + # Device-related + - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi + - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - # Pathcomp-related cleanup - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi + # Pathcomp-related + - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi + - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - # Service-related cleanup - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + # Service-related + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi script: - docker pull "cockroachdb/cockroach:latest-v22.2" @@ -76,24 +76,25 @@ unit_test service: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" # Context preparation - - | - docker volume create crdb - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 \ - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 \ - --volume "crdb:/cockroach/cockroach-data" \ + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" cockroachdb/cockroach:latest-v22.2 start-single-node - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 \ + - > + docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - docker logs nats - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + - docker logs crdb + - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done + - docker logs nats + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $NATS_ADDRESS # Context Service Preparation - | -- GitLab From 5903745dbc602cfbfef1840d495945c3cb17c932 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 18 Jul 2025 08:33:44 +0000 Subject: [PATCH 79/82] Pre-merge cleanup --- .gitlab-ci.yml | 2 +- src/device/.gitlab-ci.yml | 62 +--------------- src/service/.gitlab-ci.yml | 142 ++++++++++++------------------------- 3 files changed, 49 insertions(+), 157 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5d2bc451..7f6de0e8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,4 +59,4 @@ include: #- local: '/src/osm_client/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - - local: '/src/tests/.gitlab-ci.yml' + #- local: '/src/tests/.gitlab-ci.yml' diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 2db58fd13..d52ae4ca8 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -40,66 +40,6 @@ build device: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -## Deploy mock QKD nodes -#prepare_mock_qkd_nodes: -# stage: prepare -# before_script: -# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY -# - if docker network list | grep teraflowbridge; then echo "teraflowbridge network is already created"; else docker network create --driver=bridge teraflowbridge; fi -# - | -# # Context-related cleanup -# if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi -# if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi -# if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi -# -# script: -# - docker volume create crdb -# - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 cockroachdb/cockroach:latest-v22.2 start-single-node -# - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 -# - echo "Waiting for initialization..." -# - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done -# - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done -# - MOCK_NODES_DIR="$CI_PROJECT_DIR/controller/src/tests/tools/mock_qkd_nodes" -# - | -# if [ -d "$MOCK_NODES_DIR" ]; then -# cd "$MOCK_NODES_DIR" && ./start.sh & -# MOCK_NODES_PID=$! -# else -# echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." && exit 1; -# fi -# - echo "Waiting for mock nodes to be up..." -# - RETRY_COUNT=0 -# - MAX_RETRIES=15 -# - | -# while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do -# if curl -s http://127.0.0.1:11111 > /dev/null && \ -# curl -s http://127.0.0.1:22222 > /dev/null && \ -# curl -s http://127.0.0.1:33333 > /dev/null; then -# echo "Mock nodes are up!" -# break -# else -# echo "Mock nodes not ready, retrying in 5 seconds..." -# RETRY_COUNT=$((RETRY_COUNT + 1)) -# sleep 5 -# fi -# done -# - | -# if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then -# echo "Error: Mock nodes failed to start after multiple attempts." -# exit 1 -# fi -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -# - changes: -# - src/common/**/*.py -# - proto/*.proto -# - src/device/**/*.{py,in,yml} -# - src/device/Dockerfile -# - src/device/tests/*.py -# - src/tests/tools/mock_qkd_nodes/** -# - .gitlab-ci.yml - # Apply unit test to the component unit_test device: variables: @@ -164,7 +104,7 @@ unit_test device: # IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) # stage: deploy # needs: -# - unit test_device +# - unit_test device # # - integ_test execute # script: # - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 1ac2326b8..788a836d5 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -86,7 +86,7 @@ unit_test service: docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 nats:2.9 --http_port 8222 --user tfs --pass tfs123 - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user "tfs"'; do sleep 1; done + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - docker logs crdb - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - docker logs nats @@ -95,110 +95,62 @@ unit_test service: - echo $CRDB_ADDRESS - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $NATS_ADDRESS - - # Context Service Preparation - - | - docker run --name context -d -p 1010:1010 \ - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" \ - --env "MB_BACKEND=nats" \ - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" \ - --network=teraflowbridge \ + - > + docker run --name context -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --env "MB_BACKEND=nats" + --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" + --network=teraflowbridge $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CONTEXTSERVICE_SERVICE_HOST + - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CONTEXTSERVICE_SERVICE_HOST - # Device preparation - - | - docker run --name device -d -p 2020:2020 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --network=teraflowbridge \ + - > + docker run --name device -d -p 2020:2020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --network=teraflowbridge $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $DEVICESERVICE_SERVICE_HOST + - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $DEVICESERVICE_SERVICE_HOST - # PathComp preparation - - | - docker run --name pathcomp-backend -d -p 8081:8081 \ - --network=teraflowbridge \ + - > + docker run --name pathcomp-backend -d -p 8081:8081 + --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMP_BACKEND_HOST - sleep 1 - docker run --name pathcomp-frontend -d -p 10020:10020 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" \ - --env "PATHCOMP_BACKEND_PORT=8081" \ - --network=teraflowbridge \ + - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $PATHCOMP_BACKEND_HOST + - sleep 1 + - > + docker run --name pathcomp-frontend -d -p 10020:10020 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" + --env "PATHCOMP_BACKEND_PORT=8081" + --network=teraflowbridge $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - sleep 1 - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMPSERVICE_SERVICE_HOST + - sleep 1 + - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $PATHCOMPSERVICE_SERVICE_HOST - # Service preparation - - | - docker run --name $IMAGE_NAME -d -p 3030:3030 \ - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" \ - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" \ - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" \ - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" \ - --network=teraflowbridge \ + - > + docker run --name $IMAGE_NAME -d -p 3030:3030 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" + --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" + --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs context - docker logs device - docker logs pathcomp-frontend - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - # Mock QKD Nodes Deployment - - | - echo "Starting stage: deploy_mock_nodes" - - pip install flask # Install Flask to ensure it is available - - | - for port in 11111 22222 33333; do - if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..." - fuser -k $port/tcp - fi - done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit - ./start.sh & - MOCK_NODES_PID=$! - else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." - exit 1 - fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - if curl -s http://127.0.0.1:11111 > /dev/null && \ - curl -s http://127.0.0.1:22222 > /dev/null && \ - curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!" - break - else - echo "Mock nodes not ready, retrying in 5 seconds..." - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 5 - fi - done - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts." - exit 1 - fi + - sleep 5 + - docker ps -a + - docker logs context + - docker logs device + - docker logs pathcomp-frontend + - docker logs pathcomp-backend + - docker logs $IMAGE_NAME # Run the tests - - | - docker exec -i $IMAGE_NAME bash -c \ - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - - # Run QKD Bootstrap Test - - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: -- GitLab From c6244be4dfddec42db2f2e05bfe82013fc11d9eb Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 18 Jul 2025 08:37:51 +0000 Subject: [PATCH 80/82] Pre-merge cleanup --- src/device/.gitlab-ci.yml | 4 ++-- src/service/.gitlab-ci.yml | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index d52ae4ca8..937c37f99 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -89,7 +89,7 @@ unit_test device: - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py - src/$IMAGE_NAME/tests/Dockerfile - - src/tests/tools/mock_qkd_nodes/** + #- src/tests/tools/mock_qkd_nodes/** - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: @@ -104,7 +104,7 @@ unit_test device: # IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) # stage: deploy # needs: -# - unit_test device +# - unit test device # # - integ_test execute # script: # - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 788a836d5..34b60488b 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -105,6 +105,7 @@ unit_test service: - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CONTEXTSERVICE_SERVICE_HOST + # Device preparation - > docker run --name device -d -p 2020:2020 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" @@ -113,6 +114,7 @@ unit_test service: - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $DEVICESERVICE_SERVICE_HOST + # PathComp preparation - > docker run --name pathcomp-backend -d -p 8081:8081 --network=teraflowbridge @@ -131,6 +133,7 @@ unit_test service: - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $PATHCOMPSERVICE_SERVICE_HOST + # Service preparation - > docker run --name $IMAGE_NAME -d -p 3030:3030 --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" @@ -187,9 +190,9 @@ unit_test service: - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy service: -- GitLab From cc09274620c897bae844f6ad3ff52d1b954b2f1d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 18 Jul 2025 08:41:47 +0000 Subject: [PATCH 81/82] Pre-merge cleanup --- .gitlab-ci.yml | 56 +++++++++++++++++++------------------- src/service/.gitlab-ci.yml | 2 ++ 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f6de0e8e..ca970101f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,41 +22,41 @@ stages: # include the individual .gitlab-ci.yml of each micro-service and tests include: #- local: '/manifests/.gitlab-ci.yml' - #- local: '/src/monitoring/.gitlab-ci.yml' - #- local: '/src/nbi/.gitlab-ci.yml' - #- local: '/src/context/.gitlab-ci.yml' - #- local: '/src/device/.gitlab-ci.yml' - #- local: '/src/service/.gitlab-ci.yml' - #- local: '/src/qkd_app/.gitlab-ci.yml' - #- local: '/src/dbscanserving/.gitlab-ci.yml' - #- local: '/src/opticalattackmitigator/.gitlab-ci.yml' - #- local: '/src/opticalattackdetector/.gitlab-ci.yml' - #- local: '/src/opticalattackmanager/.gitlab-ci.yml' - #- local: '/src/opticalcontroller/.gitlab-ci.yml' - #- local: '/src/ztp/.gitlab-ci.yml' - #- local: '/src/policy/.gitlab-ci.yml' - #- local: '/src/automation/.gitlab-ci.yml' - #- local: '/src/forecaster/.gitlab-ci.yml' + - local: '/src/monitoring/.gitlab-ci.yml' + - local: '/src/nbi/.gitlab-ci.yml' + - local: '/src/context/.gitlab-ci.yml' + - local: '/src/device/.gitlab-ci.yml' + - local: '/src/service/.gitlab-ci.yml' + - local: '/src/qkd_app/.gitlab-ci.yml' + - local: '/src/dbscanserving/.gitlab-ci.yml' + - local: '/src/opticalattackmitigator/.gitlab-ci.yml' + - local: '/src/opticalattackdetector/.gitlab-ci.yml' + - local: '/src/opticalattackmanager/.gitlab-ci.yml' + - local: '/src/opticalcontroller/.gitlab-ci.yml' + - local: '/src/ztp/.gitlab-ci.yml' + - local: '/src/policy/.gitlab-ci.yml' + - local: '/src/automation/.gitlab-ci.yml' + - local: '/src/forecaster/.gitlab-ci.yml' #- local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - #- local: '/src/slice/.gitlab-ci.yml' + - local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - #- local: '/src/pathcomp/.gitlab-ci.yml' + - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - #- local: '/src/load_generator/.gitlab-ci.yml' - #- local: '/src/bgpls_speaker/.gitlab-ci.yml' - #- local: '/src/kpi_manager/.gitlab-ci.yml' - #- local: '/src/kpi_value_api/.gitlab-ci.yml' + - local: '/src/load_generator/.gitlab-ci.yml' + - local: '/src/bgpls_speaker/.gitlab-ci.yml' + - local: '/src/kpi_manager/.gitlab-ci.yml' + - local: '/src/kpi_value_api/.gitlab-ci.yml' #- local: '/src/kpi_value_writer/.gitlab-ci.yml' #- local: '/src/telemetry/.gitlab-ci.yml' - #- local: '/src/analytics/.gitlab-ci.yml' - #- local: '/src/qos_profile/.gitlab-ci.yml' - #- local: '/src/vnt_manager/.gitlab-ci.yml' - #- local: '/src/e2e_orchestrator/.gitlab-ci.yml' - #- local: '/src/ztp_server/.gitlab-ci.yml' - #- local: '/src/osm_client/.gitlab-ci.yml' + - local: '/src/analytics/.gitlab-ci.yml' + - local: '/src/qos_profile/.gitlab-ci.yml' + - local: '/src/vnt_manager/.gitlab-ci.yml' + - local: '/src/e2e_orchestrator/.gitlab-ci.yml' + - local: '/src/ztp_server/.gitlab-ci.yml' + - local: '/src/osm_client/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - #- local: '/src/tests/.gitlab-ci.yml' + - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index 34b60488b..8ceb27fdc 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -142,6 +142,8 @@ unit_test service: --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + + # Check status before the tests - sleep 5 - docker ps -a - docker logs context -- GitLab From 2dca57b0b29cf8f8f992fc6aca5cbeff7f22684f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 18 Jul 2025 08:49:11 +0000 Subject: [PATCH 82/82] QKD App & QKD SBI Driver: - Disabled pointless tests - Relocated disabled tests --- src/device/.gitlab-ci.yml | 7 +- .../qkd/unit}/test_set_new_configuration.py | 0 src/qkd_app/.gitlab-ci.yml | 170 +++++++++--------- 3 files changed, 88 insertions(+), 89 deletions(-) rename src/{qkd_app/tests => device/tests/qkd/unit}/test_set_new_configuration.py (100%) diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 937c37f99..7c22ae94c 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -71,9 +71,10 @@ unit_test device: - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_compliance.py" - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_mock_qkd_node.py" - #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose device/tests/qkd/unit/test_qkd_error_handling.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_qkd_compliance.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_mock_qkd_node.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_qkd_error_handling.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_Set_new_configuration.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: diff --git a/src/qkd_app/tests/test_set_new_configuration.py b/src/device/tests/qkd/unit/test_set_new_configuration.py similarity index 100% rename from src/qkd_app/tests/test_set_new_configuration.py rename to src/device/tests/qkd/unit/test_set_new_configuration.py diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml index 09fac2d42..41f13e876 100644 --- a/src/qkd_app/.gitlab-ci.yml +++ b/src/qkd_app/.gitlab-ci.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -build app: +build qkd_app: variables: IMAGE_NAME: 'qkd_app' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) @@ -37,88 +37,86 @@ build app: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# Apply unit test to the component -unit_test app: - variables: - IMAGE_NAME: 'qkd_app' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build app - - unit_test service - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 10070:10070 -p 8005:8005 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - sleep 5 - - docker ps -a - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - - # Mock QKD Nodes Deployment - - | - echo "Starting stage: deploy_mock_nodes" - - pip install flask # Install Flask to ensure it is available - - | - for port in 11111 22222 33333; do - if lsof -i:$port >/dev/null 2>&1; then - echo "Freeing up port $port..." - fuser -k $port/tcp - fi - done - MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" - if [ -d "$MOCK_NODES_DIR" ]; then - cd "$MOCK_NODES_DIR" || exit - ./start.sh & - MOCK_NODES_PID=$! - else - echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." - exit 1 - fi - echo "Waiting for mock nodes to be up..." - RETRY_COUNT=0 - MAX_RETRIES=15 - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - if curl -s http://127.0.0.1:11111 > /dev/null && \ - curl -s http://127.0.0.1:22222 > /dev/null && \ - curl -s http://127.0.0.1:33333 > /dev/null; then - echo "Mock nodes are up!" - break - else - echo "Mock nodes not ready, retrying in 5 seconds..." - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 5 - fi - done - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Error: Mock nodes failed to start after multiple attempts." - exit 1 - fi - - # Run additional QKD unit tests - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_create_apps.py" - - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_Set_new_configuration.py" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +## Apply unit test to the component +#unit_test qkd_app: +# variables: +# IMAGE_NAME: 'qkd_app' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build qkd_app +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker run --name $IMAGE_NAME -d -p 10070:10070 -p 8005:8005 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - sleep 5 +# - docker ps -a +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" +# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" +# +# # Mock QKD Nodes Deployment +# - | +# echo "Starting stage: deploy_mock_nodes" +# - pip install flask # Install Flask to ensure it is available +# - | +# for port in 11111 22222 33333; do +# if lsof -i:$port >/dev/null 2>&1; then +# echo "Freeing up port $port..." +# fuser -k $port/tcp +# fi +# done +# MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" +# if [ -d "$MOCK_NODES_DIR" ]; then +# cd "$MOCK_NODES_DIR" || exit +# ./start.sh & +# MOCK_NODES_PID=$! +# else +# echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." +# exit 1 +# fi +# echo "Waiting for mock nodes to be up..." +# RETRY_COUNT=0 +# MAX_RETRIES=15 +# while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do +# if curl -s http://127.0.0.1:11111 > /dev/null && \ +# curl -s http://127.0.0.1:22222 > /dev/null && \ +# curl -s http://127.0.0.1:33333 > /dev/null; then +# echo "Mock nodes are up!" +# break +# else +# echo "Mock nodes not ready, retrying in 5 seconds..." +# RETRY_COUNT=$((RETRY_COUNT + 1)) +# sleep 5 +# fi +# done +# if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then +# echo "Error: Mock nodes failed to start after multiple attempts." +# exit 1 +# fi +# +# # Run additional QKD unit tests +# - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_create_apps.py" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml -- GitLab