diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37d5fa2b54e2bd779a5ef64ebdd03cf763635a69..ca970101fd46bdf0b281d57b585089c790bc3dd8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,7 @@ include: - local: '/src/context/.gitlab-ci.yml' - local: '/src/device/.gitlab-ci.yml' - local: '/src/service/.gitlab-ci.yml' + - local: '/src/qkd_app/.gitlab-ci.yml' - local: '/src/dbscanserving/.gitlab-ci.yml' - local: '/src/opticalattackmitigator/.gitlab-ci.yml' - local: '/src/opticalattackdetector/.gitlab-ci.yml' diff --git a/deploy/all.sh b/deploy/all.sh index 93018d3ce9920bd55a53c69fb7277224e10bbf6d..a284287bc1a870d1c999e26dadc9d957b4eb974f 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -151,6 +151,26 @@ export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} export NATS_REDEPLOY=${NATS_REDEPLOY:-""} +# ----- Apache Kafka ----------------------------------------------------------- + +# If not already set, set the namespace where Kafka will be deployed. +export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} + +# If not already set, set the external port Kafka Client interface will be exposed to. +export KFK_EXT_PORT_CLIENT=${KFK_EXT_PORT_CLIENT:-"9092"} + +# If not already set, set Kafka installation mode. Accepted values are: 'single'. +# - If KFK_DEPLOY_MODE is "single", Kafka is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# NOTE: Production mode is still not supported. Will be provided in the future. +export KFK_DEPLOY_MODE=${KFK_DEPLOY_MODE:-"single"} + +# If not already set, disable flag for re-deploying Kafka from scratch. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! +# If KFK_REDEPLOY is "YES", the message broker will be dropped while checking/deploying Kafka. +export KFK_REDEPLOY=${KFK_REDEPLOY:-""} + + # ----- QuestDB ---------------------------------------------------------------- # If not already set, set the namespace where QuestDB will be deployed. diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 6866f484f0b0fd46d8ab7f196446efbd7ec6a52e..4d646194015f5c4ad1deea39ddcaa0a3a97b7f24 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -66,7 +66,7 @@ CRDB_MANIFESTS_PATH="manifests/cockroachdb" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${CRDB_NAMESPACE}/manifests" -mkdir -p $TMP_MANIFESTS_FOLDER +mkdir -p ${TMP_MANIFESTS_FOLDER} function crdb_deploy_single() { echo "CockroachDB Namespace" @@ -105,6 +105,13 @@ function crdb_deploy_single() { sleep 1 done kubectl wait --namespace ${CRDB_NAMESPACE} --for=condition=Ready --timeout=300s pod/cockroachdb-0 + + # Wait for CockroachDB to notify "start_node_query" + echo ">>> CockroachDB pods created. Waiting CockroachDB server to be started..." + while ! kubectl --namespace ${CRDB_NAMESPACE} logs pod/cockroachdb-0 -c cockroachdb 2>&1 | grep -q 'start_node_query'; do + printf "%c" "." + sleep 1 + done fi echo diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 5dee8afaecef82bf6c4a140230a155c25a17c0cf..a971c15d5401d8928f35d6c33e57d59d7d636e95 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -13,17 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. + ######################################################################################################################## # Read deployment settings ######################################################################################################################## -# If not already set, set the namespace where Apache Kafka will be deployed. +# If not already set, set the namespace where Kafka will be deployed. export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} -# If not already set, set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} +# If not already set, set the external port Kafka client interface will be exposed to. +export KFK_EXT_PORT_CLIENT=${KFK_EXT_PORT_CLIENT:-"9092"} + +# If not already set, set Kafka installation mode. Accepted values are: 'single'. +# - If KFK_DEPLOY_MODE is "single", Kafka is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# NOTE: Production mode is still not supported. Will be provided in the future. +export KFK_DEPLOY_MODE=${KFK_DEPLOY_MODE:-"single"} -# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost. +# If not already set, disable flag for re-deploying Kafka from scratch. +# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! +# If KFK_REDEPLOY is "YES", the message broker will be dropped while checking/deploying Kafka. export KFK_REDEPLOY=${KFK_REDEPLOY:-""} @@ -31,61 +40,86 @@ export KFK_REDEPLOY=${KFK_REDEPLOY:-""} # Automated steps start here ######################################################################################################################## - # Constants - TMP_FOLDER="./tmp" - KFK_MANIFESTS_PATH="manifests/kafka" - KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" - KFK_MANIFEST="02-kafka.yaml" - - # Create a tmp folder for files modified during the deployment - TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" - mkdir -p ${TMP_MANIFESTS_FOLDER} +# Constants +TMP_FOLDER="./tmp" +KFK_MANIFESTS_PATH="manifests/kafka" -function kafka_deploy() { - # copy zookeeper and kafka manifest files to temporary manifest location - cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" - cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" +mkdir -p ${TMP_MANIFESTS_FOLDER} - # echo "Apache Kafka Namespace" - echo "Delete Apache Kafka Namespace" - kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found - - echo "Create Apache Kafka Namespace" +function kfk_deploy_single() { + echo "Kafka Namespace" + echo ">>> Create Kafka Namespace (if missing)" kubectl create namespace ${KFK_NAMESPACE} + echo + + echo "Kafka (single-mode)" + echo ">>> Checking if Kafka is deployed..." + if kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; then + echo ">>> Kafka is present; skipping step." + else + echo ">>> Deploy Kafka" + cp "${KFK_MANIFESTS_PATH}/single-node.yaml" "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + #sed -i "s//${KFK_NAMESPACE}/" "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" + + echo ">>> Waiting Kafka statefulset to be created..." + while ! kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; do + printf "%c" "." + sleep 1 + done + + # Wait for statefulset condition "Available=True" does not work + # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error: + # "error: readyReplicas is not found" + # Workaround: Check the pods are ready + #echo ">>> Kafka statefulset created. Waiting for readiness condition..." + #kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/kafka + #kubectl wait --namespace ${KGK_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \ + # statefulset/kafka + echo ">>> Kafka statefulset created. Waiting Kafka pods to be created..." + while ! kubectl get --namespace ${KFK_NAMESPACE} pod/kafka-0 &> /dev/null; do + printf "%c" "." + sleep 1 + done + kubectl wait --namespace ${KFK_NAMESPACE} --for=condition=Ready --timeout=300s pod/kafka-0 + + # Wait for Kafka to notify "Kafka Server started" + echo ">>> Kafka pods created. Waiting Kafka Server to be started..." + while ! kubectl --namespace ${KFK_NAMESPACE} logs pod/kafka-0 -c kafka 2>&1 | grep -q 'Kafka Server started'; do + printf "%c" "." + sleep 1 + done + fi + echo +} - # echo ">>> Deplying Apache Kafka Zookeeper" - # Kafka zookeeper service should be deployed before the kafka service - kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" - - #KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically - #KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') - - # Kafka service should be deployed after the zookeeper service - #sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - sed -i "s//${KFK_NAMESPACE}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - - # echo ">>> Deploying Apache Kafka Broker" - kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" - # echo ">>> Verifing Apache Kafka deployment" - sleep 5 - # KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) - # if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then - # echo "Deployment Error: \n $KFK_PODS_STATUS" - # else - # echo "$KFK_PODS_STATUS" - # fi +function kfk_undeploy_single() { + echo "Kafka (single-mode)" + echo ">>> Checking if Kafka is deployed..." + if kubectl get --namespace ${KFK_NAMESPACE} statefulset/kafka &> /dev/null; then + echo ">>> Undeploy Kafka" + kubectl delete --namespace ${KFK_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/kfk_single_node.yaml" --ignore-not-found + else + echo ">>> Kafka is not present; skipping step." + fi + echo + + echo "Kafka Namespace" + echo ">>> Delete Kafka Namespace (if exists)" + echo "NOTE: this step might take few minutes to complete!" + kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found + echo } -echo ">>> Apache Kafka" -echo "Checking if Apache Kafka is deployed ... " -if [ "$KFK_REDEPLOY" == "YES" ]; then - echo "Redeploying kafka namespace" - kafka_deploy -elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then - echo "Apache Kafka already present; skipping step." +if [ "$KFK_DEPLOY_MODE" == "single" ]; then + if [ "$KFK_REDEPLOY" == "YES" ]; then + kfk_undeploy_single + fi + + kfk_deploy_single else - echo "Kafka namespace doesn't exists. Deploying kafka namespace" - kafka_deploy + echo "Unsupported value: KFK_DEPLOY_MODE=$KFK_DEPLOY_MODE" fi -echo diff --git a/deploy/tfs.sh b/deploy/tfs.sh index b73bbbf81c4c91c1ade590fb565f505d6a3c4dc3..917edb6fa86beafe864f8d2cd3cf5193fb4d4e67 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -51,12 +51,6 @@ export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""} # If not already set, set the namespace where CockroackDB will be deployed. export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"} -# If not already set, set the external port CockroackDB Postgre SQL interface will be exposed to. -export CRDB_EXT_PORT_SQL=${CRDB_EXT_PORT_SQL:-"26257"} - -# If not already set, set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. -export CRDB_EXT_PORT_HTTP=${CRDB_EXT_PORT_HTTP:-"8081"} - # If not already set, set the database username to be used by Context. export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"} @@ -69,27 +63,12 @@ export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"} # If not already set, set the namespace where NATS will be deployed. export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"} -# If not already set, set the external port NATS Client interface will be exposed to. -export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} - -# If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. -export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} - # ----- QuestDB ---------------------------------------------------------------- # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} -# If not already set, set the external port QuestDB Postgre SQL interface will be exposed to. -export QDB_EXT_PORT_SQL=${QDB_EXT_PORT_SQL:-"8812"} - -# If not already set, set the external port QuestDB Influx Line Protocol interface will be exposed to. -export QDB_EXT_PORT_ILP=${QDB_EXT_PORT_ILP:-"9009"} - -# If not already set, set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. -export QDB_EXT_PORT_HTTP=${QDB_EXT_PORT_HTTP:-"9000"} - # If not already set, set the database username to be used for QuestDB. export QDB_USERNAME=${QDB_USERNAME:-"admin"} @@ -114,14 +93,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} # ----- Apache Kafka ------------------------------------------------------ -# If not already set, set the namespace where Apache Kafka will be deployed. +# If not already set, set the namespace where Kafka will be deployed. export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} -# If not already set, set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} - -# If not already set, if flag is YES, Apache Kafka will be redeployed and topic will be lost. -export KFK_REDEPLOY=${KFK_REDEPLOY:-""} ######################################################################################################################## # Automated steps start here @@ -154,7 +128,7 @@ kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type= printf "\n" echo ">>> Create Secret with Apache Kafka..." -KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') +KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-public -o 'jsonpath={.spec.ports[0].port}') kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ --from-literal=KFK_SERVER_PORT=${KFK_SERVER_PORT} diff --git a/manifests/cockroachdb/single-node.yaml b/manifests/cockroachdb/single-node.yaml index fec9b528931019667f0be413038b9af764f33b69..ed297d77c1fbafebd303027b4ae13bd473bd219d 100644 --- a/manifests/cockroachdb/single-node.yaml +++ b/manifests/cockroachdb/single-node.yaml @@ -61,7 +61,7 @@ spec: containers: - name: cockroachdb image: cockroachdb/cockroach:latest-v22.2 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent args: - start-single-node ports: diff --git a/manifests/kafka/01-zookeeper.yaml b/manifests/kafka/01-zookeeper.yaml deleted file mode 100644 index f2cfb4f384303951983113a32680c0ad8ec65e89..0000000000000000000000000000000000000000 --- a/manifests/kafka/01-zookeeper.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - labels: - app: zookeeper-service - name: zookeeper-service -spec: - type: ClusterIP - ports: - - name: zookeeper-port - port: 2181 - #nodePort: 30181 - #targetPort: 2181 - selector: - app: zookeeper ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: zookeeper - name: zookeeper -spec: - replicas: 1 - selector: - matchLabels: - app: zookeeper - template: - metadata: - labels: - app: zookeeper - spec: - containers: - - image: wurstmeister/zookeeper - imagePullPolicy: IfNotPresent - name: zookeeper - ports: - - containerPort: 2181 diff --git a/manifests/kafka/02-kafka.yaml b/manifests/kafka/02-kafka.yaml deleted file mode 100644 index 066f0151af73ed911efdc83b627f6d74e6d9e896..0000000000000000000000000000000000000000 --- a/manifests/kafka/02-kafka.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - labels: - app: kafka-broker - name: kafka-service -spec: - ports: - - port: 9092 - selector: - app: kafka-broker ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: kafka-broker - name: kafka-broker -spec: - replicas: 1 - selector: - matchLabels: - app: kafka-broker - template: - metadata: - labels: - app: kafka-broker - spec: - hostname: kafka-broker - containers: - - env: - - name: KAFKA_BROKER_ID - value: "1" - - name: KAFKA_ZOOKEEPER_CONNECT - #value: :2181 - value: zookeeper-service..svc.cluster.local:2181 - - name: KAFKA_LISTENERS - value: PLAINTEXT://:9092 - - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka-service..svc.cluster.local:9092 - image: wurstmeister/kafka - imagePullPolicy: IfNotPresent - name: kafka-broker - ports: - - containerPort: 9092 diff --git a/manifests/kafka/single-node.yaml b/manifests/kafka/single-node.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c435c11b5478da98e1f8788dc0a6ca7df38e5f8 --- /dev/null +++ b/manifests/kafka/single-node.yaml @@ -0,0 +1,97 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: kafka-public + labels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka +spec: + type: ClusterIP + selector: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + ports: + - name: clients + port: 9092 + protocol: TCP + targetPort: 9092 + - name: control-plane + port: 9093 + protocol: TCP + targetPort: 9093 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka +spec: + selector: + matchLabels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + serviceName: "kafka-public" + replicas: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app.kubernetes.io/component: message-broker + app.kubernetes.io/instance: kafka + app.kubernetes.io/name: kafka + spec: + terminationGracePeriodSeconds: 10 + restartPolicy: Always + containers: + - name: kafka + image: bitnami/kafka:latest + imagePullPolicy: IfNotPresent + ports: + - name: clients + containerPort: 9092 + - name: control-plane + containerPort: 9093 + env: + #- name: KAFKA_BROKER_ID + # value: "1" + #- name: KAFKA_ZOOKEEPER_CONNECT + # value: zookeeper-service..svc.cluster.local:2181 + #- name: KAFKA_LISTENERS + # value: PLAINTEXT://:9092 + #- name: KAFKA_ADVERTISED_LISTENERS + # value: PLAINTEXT://kafka-service..svc.cluster.local:9092 + - name: KAFKA_CFG_NODE_ID + value: "1" + - name: KAFKA_CFG_PROCESS_ROLES + value: "controller,broker" + - name: KAFKA_CFG_LISTENERS + value: "PLAINTEXT://:9092,CONTROLLER://:9093" + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT" + - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES + value: "CONTROLLER" + - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS + value: "1@kafka-0:9093" + resources: + requests: + cpu: "250m" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index cead19406afd01718f8f3d105fabd3cc4754356b..27026cc0f8864d012986fb9f7e5c547125930570 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -41,7 +41,7 @@ spec: - name: LOG_LEVEL value: "INFO" - name: FLASK_ENV - value: "production" # change to "development" if developing + value: "production" # normal value is "production", change to "development" if developing - name: IETF_NETWORK_RENDERER value: "LIBYANG" envFrom: diff --git a/my_deploy.sh b/my_deploy.sh index 4d3820f41affacdb5aea743e3f4cedc310442a05..662dc389b123daabe02bedf2f43232edde8f3bc3 100644 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -159,6 +159,22 @@ export NATS_DEPLOY_MODE="single" export NATS_REDEPLOY="" +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + +# Set the port Apache Kafka server will be exposed to. +export KFK_EXT_PORT_CLIENT="9092" + +# Set Kafka installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/kafka.sh for additional details +export KFK_DEPLOY_MODE="single" + +# Disable flag for re-deploying Kafka from scratch. +export KFK_REDEPLOY="" + + # ----- QuestDB ---------------------------------------------------------------- # Set the namespace where QuestDB will be deployed. @@ -199,15 +215,3 @@ export PROM_EXT_PORT_HTTP="9090" # Set the external port Grafana HTTP Dashboards will be exposed to. export GRAF_EXT_PORT_HTTP="3000" - - -# ----- Apache Kafka ----------------------------------------------------------- - -# Set the namespace where Apache Kafka will be deployed. -export KFK_NAMESPACE="kafka" - -# Set the port Apache Kafka server will be exposed to. -export KFK_SERVER_PORT="9092" - -# Set the flag to YES for redeploying of Apache Kafka -export KFK_REDEPLOY="" diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index 3ca23d080863dba8472be27a4910abd9ead93894..9337fd7123a221cc4199bb33a69280c11bb6efc5 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -69,19 +69,19 @@ def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: def format_device_custom_config_rules(device : Dict) -> Dict: config_rules = device.get('device_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - device['device_config']['config_rules'] = config_rules + device.setdefault('device_config', {})['config_rules'] = config_rules return device def format_service_custom_config_rules(service : Dict) -> Dict: config_rules = service.get('service_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - service['service_config']['config_rules'] = config_rules + service.setdefault('service_config', {})['config_rules'] = config_rules return service def format_slice_custom_config_rules(slice_ : Dict) -> Dict: config_rules = slice_.get('slice_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - slice_['slice_config']['config_rules'] = config_rules + slice_.setdefault('slice_config', {})['config_rules'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: @@ -138,6 +138,19 @@ def link_type_to_str(link_type : Union[int, str]) -> Optional[str]: if isinstance(link_type, str): return LinkTypeEnum.Name(LinkTypeEnum.Value(link_type)) return None +LINK_TYPES_NORMAL = { + LinkTypeEnum.LINKTYPE_UNKNOWN, + LinkTypeEnum.LINKTYPE_COPPER, + LinkTypeEnum.LINKTYPE_RADIO, + LinkTypeEnum.LINKTYPE_MANAGEMENT, +} +LINK_TYPES_OPTICAL = { + LinkTypeEnum.LINKTYPE_FIBER, +} +LINK_TYPES_VIRTUAL = { + LinkTypeEnum.LINKTYPE_VIRTUAL, +} + def split_links_by_type(links : List[Dict]) -> Dict[str, List[Dict]]: typed_links = collections.defaultdict(list) for link in links: @@ -148,11 +161,11 @@ def split_links_by_type(links : List[Dict]) -> Dict[str, List[Dict]]: raise Exception(MSG.format(str(link))) link_type = LinkTypeEnum.Value(str_link_type) - if link_type in {LinkTypeEnum.LINKTYPE_UNKNOWN, LinkTypeEnum.LINKTYPE_COPPER, LinkTypeEnum.LINKTYPE_RADIO, LinkTypeEnum.LINKTYPE_MANAGEMENT}: + if link_type in LINK_TYPES_NORMAL: typed_links['normal'].append(link) - elif link_type in {LinkTypeEnum.LINKTYPE_FIBER}: + elif link_type in LINK_TYPES_OPTICAL: typed_links['optical'].append(link) - elif link_type in {LinkTypeEnum.LINKTYPE_VIRTUAL}: + elif link_type in LINK_TYPES_VIRTUAL: typed_links['virtual'].append(link) else: MSG = 'Unsupported LinkType({:s}) in Link({:s})' diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index eac8dfc27783fa2399c4e2ab73793e19c31987ce..7bb131dd6c76789b07c44c9568ede038af1e4d45 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -20,7 +20,7 @@ from common.Settings import get_setting LOGGER = logging.getLogger(__name__) -KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' +KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-public.{:s}.svc.cluster.local:{:s}' KAFKA_TOPIC_NUM_PARTITIONS = 1 KAFKA_TOPIC_REPLICATION_FACTOR = 1 diff --git a/src/device/.gitlab-ci.yml b/src/device/.gitlab-ci.yml index 6be3b5bdf8149df50d8d4d165d8b8277f259fb48..7c22ae94cd73b0e7488904a05254d590e95e7511 100644 --- a/src/device/.gitlab-ci.yml +++ b/src/device/.gitlab-ci.yml @@ -40,30 +40,6 @@ build device: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -## Start Mock QKD Nodes before unit testing -#start_mock_nodes: -# stage: deploy -# script: -# - bash src/tests/tools/mock_qkd_nodes/start.sh & -# - sleep 10 # wait for nodes to spin up -# artifacts: -# paths: -# - mock_nodes.log -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - -## Prepare Scenario (Start NBI, mock services) -#prepare_scenario: -# stage: deploy -# script: -# - pytest src/tests/qkd/unit/PrepareScenario.py -# needs: -# - start_mock_nodes -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - # Apply unit test to the component unit_test device: variables: @@ -72,8 +48,6 @@ unit_test device: stage: unit_test needs: - build device - #- start_mock_nodes - #- prepare_scenario before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - > @@ -97,6 +71,10 @@ unit_test device: - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_emulated.py --junitxml=/opt/results/${IMAGE_NAME}_report_emulated.xml" - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary_ietf_actn.py --junitxml=/opt/results/${IMAGE_NAME}_report_ietf_actn.xml" #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_*.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_qkd_compliance.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_mock_qkd_node.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_qkd_error_handling.py" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/qkd/unit/test_Set_new_configuration.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: @@ -112,12 +90,13 @@ unit_test device: - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py - src/$IMAGE_NAME/tests/Dockerfile + #- src/tests/tools/mock_qkd_nodes/** - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml ## Deployment of the service in Kubernetes Cluster #deploy device: diff --git a/src/device/service/drivers/qkd/Tools2.py b/src/device/service/drivers/qkd/Tools2.py index f07a1324a3d759906a1d5cc7b4309260777a71af..ab0a140a310bda604c0f0c2a0cef3d0a4f605ae3 100644 --- a/src/device/service/drivers/qkd/Tools2.py +++ b/src/device/service/drivers/qkd/Tools2.py @@ -224,11 +224,16 @@ def fetch_node(url: str, resource_key: str, headers: Dict[str, str], auth: Optio try: r = requests.get(url, timeout=timeout, verify=False, auth=auth, headers=headers) r.raise_for_status() - result.append((resource_key, r.json().get('qkd_node', {}))) + data = r.json() + data.pop('qkdn_capabilities', None) + data.pop('qkd_applications', None) + data.pop('qkd_interfaces', None) + data.pop('qkd_links', None) + result.append((resource_key, data)) except requests.RequestException as e: LOGGER.error(f"Error fetching node from {url}: {e}") result.append((resource_key, e)) - + return result diff --git a/src/device/tests/qkd/unit/test_qkd_compliance.py b/src/device/tests/qkd/unit/test_qkd_compliance.py index e0cfe90cdfab2a060cd4b5ae968583af1da750de..4dec4ab6a35a8618947355e29e69c5cd780728e2 100644 --- a/src/device/tests/qkd/unit/test_qkd_compliance.py +++ b/src/device/tests/qkd/unit/test_qkd_compliance.py @@ -15,10 +15,18 @@ import pytest import requests -from tests.tools.mock_qkd_nodes.YangValidator import YangValidator +from requests.exceptions import HTTPError +from tests.tools.mock_qkd_node.YangValidator import YangValidator def test_compliance_with_yang_models(): validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') - data = response.json() - assert validator.parse_to_dict(data) is not None + try: + response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') + response.raise_for_status() + data = response.json() + assert validator.parse_to_dict(data) is not None, "Data validation failed against YANG model." + except HTTPError as e: + pytest.fail(f"HTTP error occurred: {e}") + except Exception as e: + pytest.fail(f"Unexpected error occurred: {e}") + diff --git a/src/device/tests/qkd/unit/test_qkd_error_hanling.py b/src/device/tests/qkd/unit/test_qkd_error_handling.py similarity index 96% rename from src/device/tests/qkd/unit/test_qkd_error_hanling.py rename to src/device/tests/qkd/unit/test_qkd_error_handling.py index 5d847ac381d34dd9457591cd416da5040fe8b115..4d674f109c0cd4fdf148f1297b89f6511a6e87d4 100644 --- a/src/device/tests/qkd/unit/test_qkd_error_hanling.py +++ b/src/device/tests/qkd/unit/test_qkd_error_handling.py @@ -40,7 +40,7 @@ def test_invalid_operations_on_network_links(qkd_driver): try: # Attempt to perform an invalid operation (simulate wrong resource key) - response = requests.post(f'http://{qkd_driver.address}/invalid_resource', json=invalid_payload) + response = requests.post(f'http://{qkd_driver.address}:{qkd_driver.port}/invalid_resource', json=invalid_payload) response.raise_for_status() except HTTPError as e: diff --git a/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py b/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py index 05b589e3882f08e22959fc57383f0a57619cb32b..a2d59fcb3e14a608810d1b4770f2525a8ab41acc 100644 --- a/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py +++ b/src/device/tests/qkd/unit/test_qkd_mock_connectivity.py @@ -12,16 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest, requests +import pytest +import requests +import time +import socket from unittest.mock import patch -from device.service.drivers.qkd.QKDDriver import QKDDriver +from device.service.drivers.qkd.QKDDriver2 import QKDDriver -MOCK_QKD_ADDRRESS = '127.0.0.1' +MOCK_QKD_ADDRESS = '127.0.0.1' # Use localhost to connect to the mock node in the Docker container MOCK_PORT = 11111 +@pytest.fixture(scope="module") +def wait_for_mock_node(): + """ + Fixture to wait for the mock QKD node to be ready before running tests. + """ + timeout = 30 # seconds + start_time = time.time() + while True: + try: + with socket.create_connection((MOCK_QKD_ADDRESS, MOCK_PORT), timeout=1): + break # Success + except (socket.timeout, socket.error): + if time.time() - start_time > timeout: + raise RuntimeError("Timed out waiting for mock QKD node to be ready.") + time.sleep(1) + @pytest.fixture -def qkd_driver(): - return QKDDriver(address=MOCK_QKD_ADDRRESS, port=MOCK_PORT, username='user', password='pass') +def qkd_driver(wait_for_mock_node): + return QKDDriver(address=MOCK_QKD_ADDRESS, port=MOCK_PORT, username='user', password='pass') # Deliverable Test ID: SBI_Test_01 def test_qkd_driver_connection(qkd_driver): @@ -29,7 +48,7 @@ def test_qkd_driver_connection(qkd_driver): # Deliverable Test ID: SBI_Test_01 def test_qkd_driver_invalid_connection(): - qkd_driver = QKDDriver(address='127.0.0.1', port=12345, username='user', password='pass') # Use invalid port directly + qkd_driver = QKDDriver(address=MOCK_QKD_ADDRESS, port=12345, username='user', password='pass') # Use invalid port directly assert qkd_driver.Connect() is False # Deliverable Test ID: SBI_Test_10 @@ -38,4 +57,3 @@ def test_qkd_driver_timeout_connection(mock_get, qkd_driver): mock_get.side_effect = requests.exceptions.Timeout qkd_driver.timeout = 0.001 # Simulate very short timeout assert qkd_driver.Connect() is False - diff --git a/src/device/tests/qkd/unit/test_set_new_configuration.py b/src/device/tests/qkd/unit/test_set_new_configuration.py index 3515f458494ac1b5616fc0ff7b12f3031c0aea53..1b5dfa2ba441b6f0d60d127d7f7acf6170d6d2da 100644 --- a/src/device/tests/qkd/unit/test_set_new_configuration.py +++ b/src/device/tests/qkd/unit/test_set_new_configuration.py @@ -53,7 +53,7 @@ def create_qkd_app(driver, qkdn_id, backing_qkdl_id, client_app_id=None): print(f"Sending payload to {driver.address}: {app_payload}") # Send POST request to create the application - response = requests.post(f'http://{driver.address}/app/create_qkd_app', json=app_payload) + response = requests.post(f'http://{driver.address}/qkd_app/create_qkd_app', json=app_payload) # Check if the request was successful (HTTP 2xx) response.raise_for_status() diff --git a/src/nbi/service/qkd_app/Resources.py b/src/nbi/service/qkd_app/Resources.py index fb4ec45d4700260fd3211a332f74954100aaf4c9..2f391ce41d4dd86b20a93e6779ea2dc5dd7cc416 100644 --- a/src/nbi/service/qkd_app/Resources.py +++ b/src/nbi/service/qkd_app/Resources.py @@ -12,16 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid -import json +import json, logging from flask import request +from flask.json import jsonify from flask_restful import Resource from common.proto.context_pb2 import Empty from common.proto.qkd_app_pb2 import App, QKDAppTypesEnum from common.Constants import DEFAULT_CONTEXT_NAME from context.client.ContextClient import ContextClient +from nbi.service._tools.HttpStatusCodes import HTTP_OK, HTTP_SERVERERROR from qkd_app.client.QKDAppClient import QKDAppClient +LOGGER = logging.getLogger(__name__) + class _Resource(Resource): def __init__(self) -> None: super().__init__() @@ -30,7 +33,7 @@ class _Resource(Resource): class Index(_Resource): def get(self): - return {'hello': 'world'} + return {} class ListDevices(_Resource): def get(self): @@ -79,20 +82,35 @@ class CreateQKDApp(_Resource): def post(self): app = request.get_json()['app'] devices = self.context_client.ListDevices(Empty()).devices - local_device = None + + local_qkdn_id = app.get('local_qkdn_id') + if local_qkdn_id is None: + MSG = 'local_qkdn_id not specified in qkd_app({:s})' + msg = MSG.format(str(app)) + LOGGER.exception(msg) + response = jsonify({'error': msg}) + response.status_code = HTTP_SERVERERROR + return response # This for-loop won't be necessary if Device ID is guaranteed to be the same as QKDN Id + local_device = None for device in devices: for config_rule in device.device_config.config_rules: - if config_rule.custom.resource_key == '__node__': - value = json.loads(config_rule.custom.resource_value) - qkdn_id = value['qkdn_id'] - if app['local_qkdn_id'] == qkdn_id: - local_device = device - break + if config_rule.custom.resource_key != '__node__': continue + value = json.loads(config_rule.custom.resource_value) + qkdn_id = value.get('qkdn_id') + if qkdn_id is None: continue + if local_qkdn_id != qkdn_id: continue + local_device = device + break if local_device is None: - return {"status": "fail"} + MSG = 'Unable to find device for local_qkdn_id({:s})' + msg = MSG.format(str(local_qkdn_id)) + LOGGER.exception(msg) + response = jsonify({'error': msg}) + response.status_code = HTTP_SERVERERROR + return response external_app_src_dst = { 'app_id': {'context_id': {'context_uuid': {'uuid': DEFAULT_CONTEXT_NAME}}, 'app_uuid': {'uuid': ''}}, @@ -107,5 +125,6 @@ class CreateQKDApp(_Resource): self.qkd_app_client.RegisterApp(App(**external_app_src_dst)) - return {"status": "success"} - + response = jsonify({'status': 'success'}) + response.status_code = HTTP_OK + return response diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml index 5bba29ca5a31f0f32b70f9bf2be996cf05cd1b4e..41f13e876641f8abf5176f1ae83772b1694ef062 100644 --- a/src/qkd_app/.gitlab-ci.yml +++ b/src/qkd_app/.gitlab-ci.yml @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -build app: +build qkd_app: variables: IMAGE_NAME: 'qkd_app' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) @@ -37,44 +37,86 @@ build app: - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml -# Apply unit test to the component -unit_test app: - variables: - IMAGE_NAME: 'qkd_app' # name of the microservice - IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) - stage: unit_test - needs: - - build app - - unit_test service - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - script: - - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 10070:10070 -p 8005:8005 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - sleep 5 - - docker ps -a - - docker logs $IMAGE_NAME - - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - after_script: - - docker rm -f $IMAGE_NAME - - docker network rm teraflowbridge - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - - changes: - - src/common/**/*.py - - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} - - src/$IMAGE_NAME/Dockerfile - - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile - - manifests/${IMAGE_NAME}service.yaml - - .gitlab-ci.yml - artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml +## Apply unit test to the component +#unit_test qkd_app: +# variables: +# IMAGE_NAME: 'qkd_app' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: unit_test +# needs: +# - build qkd_app +# before_script: +# - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi +# - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi +# script: +# - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" +# - docker run --name $IMAGE_NAME -d -p 10070:10070 -p 8005:8005 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG +# - sleep 5 +# - docker ps -a +# - docker logs $IMAGE_NAME +# - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" +# - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" +# +# # Mock QKD Nodes Deployment +# - | +# echo "Starting stage: deploy_mock_nodes" +# - pip install flask # Install Flask to ensure it is available +# - | +# for port in 11111 22222 33333; do +# if lsof -i:$port >/dev/null 2>&1; then +# echo "Freeing up port $port..." +# fuser -k $port/tcp +# fi +# done +# MOCK_NODES_DIR="$PWD/src/tests/tools/mock_qkd_nodes" +# if [ -d "$MOCK_NODES_DIR" ]; then +# cd "$MOCK_NODES_DIR" || exit +# ./start.sh & +# MOCK_NODES_PID=$! +# else +# echo "Error: Mock QKD nodes directory '$MOCK_NODES_DIR' not found." +# exit 1 +# fi +# echo "Waiting for mock nodes to be up..." +# RETRY_COUNT=0 +# MAX_RETRIES=15 +# while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do +# if curl -s http://127.0.0.1:11111 > /dev/null && \ +# curl -s http://127.0.0.1:22222 > /dev/null && \ +# curl -s http://127.0.0.1:33333 > /dev/null; then +# echo "Mock nodes are up!" +# break +# else +# echo "Mock nodes not ready, retrying in 5 seconds..." +# RETRY_COUNT=$((RETRY_COUNT + 1)) +# sleep 5 +# fi +# done +# if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then +# echo "Error: Mock nodes failed to start after multiple attempts." +# exit 1 +# fi +# +# # Run additional QKD unit tests +# - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_create_apps.py" +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# after_script: +# - docker rm -f $IMAGE_NAME +# - docker network rm teraflowbridge +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# - changes: +# - src/common/**/*.py +# - proto/*.proto +# - src/$IMAGE_NAME/**/*.{py,in,yml} +# - src/$IMAGE_NAME/Dockerfile +# - src/$IMAGE_NAME/tests/*.py +# - src/$IMAGE_NAME/tests/Dockerfile +# - manifests/${IMAGE_NAME}service.yaml +# - .gitlab-ci.yml +# artifacts: +# when: always +# reports: +# junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml diff --git a/src/qkd_app/service/QKDAppServiceServicerImpl.py b/src/qkd_app/service/QKDAppServiceServicerImpl.py index 4179d4047d74bdbdc31cf394f36903f28c96cf45..c58b0abaf17a0c1c6519f49ac30dd4b6e7452c49 100644 --- a/src/qkd_app/service/QKDAppServiceServicerImpl.py +++ b/src/qkd_app/service/QKDAppServiceServicerImpl.py @@ -135,52 +135,22 @@ class AppServiceServicerImpl(AppServiceServicer): app_set(self.db_engine, self.messagebroker, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def ListApps(self, request: ContextId, context: grpc.ServicerContext) -> AppList: + def ListApps(self, request : ContextId, context : grpc.ServicerContext) -> AppList: """ Lists all apps in the system, including their statistics and QoS attributes. """ - LOGGER.debug(f"Received ListApps request: {grpc_message_to_json_string(request)}") - - try: - apps = app_list_objs(self.db_engine, request.context_uuid.uuid) - for app in apps.apps: - LOGGER.debug(f"App retrieved: {grpc_message_to_json_string(app)}") - - LOGGER.debug(f"ListApps returned {len(apps.apps)} apps for context_id: {request.context_uuid.uuid}") - return apps - except Exception as e: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details("An internal error occurred while listing apps.") - raise e + return app_list_objs(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetApp(self, request: AppId, context: grpc.ServicerContext) -> App: + def GetApp(self, request : AppId, context : grpc.ServicerContext) -> App: """ Fetches details of a specific app based on its AppId, including QoS and performance stats. """ - LOGGER.debug(f"Received GetApp request: {grpc_message_to_json_string(request)}") - try: - app = app_get(self.db_engine, request) - LOGGER.debug(f"GetApp found app with app_uuid: {request.app_uuid.uuid}") - return app - except NotFoundException as e: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details(f"App not found: {e}") - raise e + return app_get(self.db_engine, request) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def DeleteApp(self, request: AppId, context: grpc.ServicerContext) -> Empty: + def DeleteApp(self, request : AppId, context : grpc.ServicerContext) -> Empty: """ Deletes an app from the system by its AppId, following ETSI compliance. """ - LOGGER.debug(f"Received DeleteApp request for app_uuid: {request.app_uuid.uuid}") - try: - app_delete(self.db_engine, request.app_uuid.uuid) - LOGGER.debug(f"App with UUID {request.app_uuid.uuid} deleted successfully.") - return Empty() - except NotFoundException as e: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details(f"App not found: {e}") - raise e - - + return app_delete(self.db_engine, request) diff --git a/src/qkd_app/service/database/QKDApp.py b/src/qkd_app/service/database/QKDApp.py index c724ef6607d3cf4eea92c2ecb6ea515357475430..00d9cb7ac849cf1661695b9b99ca0e5089c9dfde 100644 --- a/src/qkd_app/service/database/QKDApp.py +++ b/src/qkd_app/service/database/QKDApp.py @@ -23,9 +23,8 @@ from sqlalchemy_cockroachdb import run_transaction from common.method_wrappers.ServiceExceptions import NotFoundException from common.message_broker.MessageBroker import MessageBroker +from common.proto.context_pb2 import ContextId, Empty from common.proto.qkd_app_pb2 import AppList, App, AppId -from qkd_app.service.database.uuids._Builder import get_uuid_from_string, get_uuid_random -from common.method_wrappers.ServiceExceptions import InvalidArgumentsException from common.tools.object_factory.QKDApp import json_app_id from common.tools.object_factory.Context import json_context_id @@ -38,27 +37,25 @@ from .models.enums.QKDAppTypes import grpc_to_enum__qkd_app_types LOGGER = logging.getLogger(__name__) -def app_list_objs(db_engine: Engine, context_uuid: str = None) -> AppList: +def app_list_objs(db_engine : Engine, request : ContextId) -> AppList: """ Fetches a list of all QKD applications from the database. Optionally filters by context UUID. :param db_engine: SQLAlchemy Engine for DB connection - :param context_uuid: UUID of the context to filter by (optional) + :param request: Context Id containing the UUID of the context to filter by :return: AppList containing all apps """ - def callback(session: Session) -> List[Dict]: + context_uuid = context_get_uuid(request, allow_random=False) + def callback(session : Session) -> List[Dict]: query = session.query(AppModel) - - if context_uuid: - query = query.filter_by(context_uuid=context_uuid) - - return [obj.dump() for obj in query.all()] - + query = query.filter_by(context_uuid=context_uuid) + obj_list : List[AppModel] = query.all() + return [obj.dump() for obj in obj_list] apps = run_transaction(sessionmaker(bind=db_engine), callback) return AppList(apps=apps) -def app_get(db_engine: Engine, request: AppId) -> App: +def app_get(db_engine : Engine, request : AppId) -> App: """ Fetches a specific app by its UUID. @@ -67,23 +64,27 @@ def app_get(db_engine: Engine, request: AppId) -> App: :return: App protobuf object :raises NotFoundException: If the app is not found in the database """ - app_uuid = app_get_uuid(request, allow_random=False) + context_uuid,app_uuid = app_get_uuid(request, allow_random=False) - def callback(session: Session) -> Optional[Dict]: - obj = session.query(AppModel).filter_by(app_uuid=app_uuid).one_or_none() - return obj.dump() if obj else None + def callback(session : Session) -> Optional[Dict]: + query = session.query(AppModel) + query = query.filter_by(app_uuid=app_uuid) + obj : Optional[AppModel] = query.one_or_none() + return None if obj is None else obj.dump() obj = run_transaction(sessionmaker(bind=db_engine), callback) - - if not obj: - raise NotFoundException('App', request.app_uuid.uuid, extra_details=[ - f'app_uuid generated was: {app_uuid}' + + if obj is None: + raw_app_uuid = '{:s}/{:s}'.format(request.context_id.context_uuid.uuid, request.app_uuid.uuid) + raise NotFoundException('App', raw_app_uuid, extra_details=[ + 'context_uuid generated was: {:s}'.format(context_uuid), + 'app_uuid generated was: {:s}'.format(app_uuid), ]) - + return App(**obj) -def app_set(db_engine: Engine, messagebroker: MessageBroker, request: App) -> AppId: +def app_set(db_engine : Engine, messagebroker : MessageBroker, request : App) -> AppId: """ Creates or updates an app in the database. If the app already exists, updates the app. Otherwise, inserts a new entry. @@ -93,8 +94,7 @@ def app_set(db_engine: Engine, messagebroker: MessageBroker, request: App) -> Ap :param request: App protobuf object containing app data :return: AppId protobuf object representing the newly created or updated app """ - context_uuid = context_get_uuid(request.app_id.context_id, allow_random=False) - app_uuid = app_get_uuid(request.app_id, allow_random=True) + context_uuid,app_uuid = app_get_uuid(request.app_id, allow_random=True) # Prepare app data for insertion/update app_data = { @@ -154,20 +154,21 @@ def app_get_by_server(db_engine: Engine, server_app_id: str) -> App: return App(**obj) -def app_delete(db_engine: Engine, app_uuid: str) -> None: +def app_delete(db_engine : Engine, request : AppId) -> Empty: """ Deletes an app by its UUID from the database. :param db_engine: SQLAlchemy Engine for DB connection - :param app_uuid: The UUID of the app to be deleted + :param app_id: The UUID of the app to be deleted """ - def callback(session: Session) -> bool: - app_obj = session.query(AppModel).filter_by(app_uuid=app_uuid).one_or_none() - if app_obj is None: - raise NotFoundException('App', app_uuid) + _,app_uuid = app_get_uuid(request, allow_random=False) - session.delete(app_obj) - return True + def callback(session : Session) -> bool: + query = session.query(AppModel) + query = query.filter_by(app_uuid=app_uuid) + num_deleted = query.delete() + return num_deleted > 0 run_transaction(sessionmaker(bind=db_engine), callback) + return Empty() diff --git a/src/qkd_app/service/database/uuids/Context.py b/src/qkd_app/service/database/uuids/Context.py new file mode 100644 index 0000000000000000000000000000000000000000..752e62efec9b7378216ae523fd1d4104321f4496 --- /dev/null +++ b/src/qkd_app/service/database/uuids/Context.py @@ -0,0 +1,37 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.method_wrappers.ServiceExceptions import InvalidArgumentsException +from ._Builder import get_uuid_from_string, get_uuid_random + +def context_get_uuid( + context_id : ContextId, context_name : str = '', allow_random : bool = False, allow_default : bool = False +) -> str: + context_uuid = context_id.context_uuid.uuid + + if len(context_uuid) > 0: + return get_uuid_from_string(context_uuid) + if len(context_name) > 0: + return get_uuid_from_string(context_name) + if allow_default: + return get_uuid_from_string(DEFAULT_CONTEXT_NAME) + if allow_random: + return get_uuid_random() + + raise InvalidArgumentsException([ + ('context_id.context_uuid.uuid', context_uuid), + ('name', context_name), + ], extra_details=['At least one is required to produce a Context UUID']) diff --git a/src/qkd_app/service/database/uuids/QKDApp.py b/src/qkd_app/service/database/uuids/QKDApp.py index 86f33f58e32556b204eeb1a4c814474af6430132..135ac9ec6faa9a6c3a2f8466497370eb8b66d774 100644 --- a/src/qkd_app/service/database/uuids/QKDApp.py +++ b/src/qkd_app/service/database/uuids/QKDApp.py @@ -12,26 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Tuple from common.proto.qkd_app_pb2 import AppId from common.method_wrappers.ServiceExceptions import InvalidArgumentsException from ._Builder import get_uuid_from_string, get_uuid_random +from .Context import context_get_uuid -def app_get_uuid(app_id: AppId, allow_random: bool = False) -> str: +def app_get_uuid( + app_id : AppId, app_name : str = '', allow_random : bool = False +) -> Tuple[str, str]: """ Retrieves or generates the UUID for an app. - :param app_id: AppId object that contains the app UUID + :param application_id: AppId object that contains the app UUID + :param application_name: string that contains optional app name :param allow_random: If True, generates a random UUID if app_uuid is not set - :return: App UUID as a string + :return: Context UUID as a string , App UUID as a string """ - app_uuid = app_id.app_uuid.uuid + context_uuid = context_get_uuid(app_id.context_id, allow_random=False, allow_default=True) + raw_app_uuid = app_id.app_uuid.uuid + + if len(raw_app_uuid) > 0: + return context_uuid, get_uuid_from_string(raw_app_uuid, prefix_for_name=context_uuid) + + if len(app_name) > 0: + return context_uuid, get_uuid_from_string(app_name, prefix_for_name=context_uuid) - if app_uuid: - return get_uuid_from_string(app_uuid) - if allow_random: - return get_uuid_random() + return context_uuid, get_uuid_random() raise InvalidArgumentsException([ - ('app_id.app_uuid.uuid', app_uuid), - ], extra_details=['At least one UUID is required to identify the app.']) + ('app_id.app_uuid.uuid', raw_app_uuid), + ('name', app_name), + ], extra_details=['At least one is required to produce a App UUID']) diff --git a/src/device/tests/qkd/unit/test_create_apps.py b/src/qkd_app/tests/test_create_apps.py similarity index 67% rename from src/device/tests/qkd/unit/test_create_apps.py rename to src/qkd_app/tests/test_create_apps.py index 557f8c23a8f976ea14d88834a2cd4e2f50cc1035..c16c319068ba3951a8b859fbf35aec4887484aa6 100644 --- a/src/device/tests/qkd/unit/test_create_apps.py +++ b/src/qkd_app/tests/test_create_apps.py @@ -14,7 +14,7 @@ import requests -QKD_ADDRESS = '10.0.2.10' +QKD_ADDRESS = '127.0.0.1' QKD_URL = 'http://{:s}/qkd_app/create_qkd_app'.format(QKD_ADDRESS) QKD_REQUEST_1 = { @@ -22,19 +22,21 @@ QKD_REQUEST_1 = { 'server_app_id': '1', 'client_app_id': [], 'app_status': 'ON', - 'local_qkdn_id': '00000001-0000-0000-0000-0000000000', - 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + 'local_qkdn_id': '00000001-0000-0000-0000-000000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-000000000000'], } } -print(requests.post(QKD_URL, json=QKD_REQUEST_1)) +reply = requests.post(QKD_URL, json=QKD_REQUEST_1) +print(reply.status_code, reply.text) QKD_REQUEST_2 = { 'app': { 'server_app_id': '1', 'client_app_id': [], 'app_status': 'ON', - 'local_qkdn_id': '00000003-0000-0000-0000-0000000000', - 'backing_qkdl_id': ['00000003-0002-0000-0000-0000000000'] + 'local_qkdn_id': '00000003-0000-0000-0000-000000000000', + 'backing_qkdl_id': ['00000003-0002-0000-0000-000000000000'], } } -print(requests.post(QKD_URL, json=QKD_REQUEST_2)) +reply = requests.post(QKD_URL, json=QKD_REQUEST_2) +print(reply.status_code, reply.text) diff --git a/src/service/.gitlab-ci.yml b/src/service/.gitlab-ci.yml index b8ca2c14377e88170e2628843b17aab388362e86..8ceb27fdc53e111208322a80ee49e3f3d337bbf3 100644 --- a/src/service/.gitlab-ci.yml +++ b/src/service/.gitlab-ci.yml @@ -153,9 +153,8 @@ unit_test service: - docker logs $IMAGE_NAME # Run the tests - - > - docker exec -i $IMAGE_NAME bash -c - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + #- docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose service/tests/qkd/test_functional_bootstrap.py" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' @@ -193,9 +192,9 @@ unit_test service: - .gitlab-ci.yml artifacts: - when: always - reports: - junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster #deploy service: diff --git a/src/device/tests/qkd/unit/descriptorQKD_links.json b/src/service/tests/qkd/descriptorQKD_links.json similarity index 68% rename from src/device/tests/qkd/unit/descriptorQKD_links.json rename to src/service/tests/qkd/descriptorQKD_links.json index 28a9e7d5ae014f78cfa0e554ee73a53449bba03c..d80864cb0bfd8ee1fed11a6af482f50620953894 100644 --- a/src/device/tests/qkd/unit/descriptorQKD_links.json +++ b/src/service/tests/qkd/descriptorQKD_links.json @@ -10,68 +10,64 @@ "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "11111"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - }, { "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "22222"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - }, - { + { "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_operational_status": 0, "device_drivers": [12], "device_endpoints": [], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.0.2.10"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": ""}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "33333"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "scheme": "http" }}} ]} - } ], "links": [ - { - "link_id": {"link_uuid": {"uuid": "QKD1/10.0.2.10:1001==QKD2/10.0.2.10:2001"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD1/:1001==QKD2/:2001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}} + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": ":1001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2001"}} ] }, { - "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2001==QKD1/10.0.2.10:1001"}}, + "link_id": {"link_uuid": {"uuid": "QKD2/:2001==QKD1/:1001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "10.0.2.10:1001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": ":1001"}} ] }, - { - "link_id": {"link_uuid": {"uuid": "QKD2/10.0.2.10:2002==QKD3/10.0.2.10:3001"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD2/:2002==QKD3/:3001"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}}, - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}} + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2002"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": ":3001"}} ] }, - { - "link_id": {"link_uuid": {"uuid": "QKD3/10.0.2.10:3001==QKD2/10.0.2.10:2002"}}, + { + "link_id": {"link_uuid": {"uuid": "QKD3/:3001==QKD2/:2002"}}, "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "10.0.2.10:3001"}}, - {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "10.0.2.10:2002"}} + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": ":3001"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": ":2002"}} ] } - ] -} +} \ No newline at end of file diff --git a/src/service/tests/qkd/test_functional_bootstrap.py b/src/service/tests/qkd/test_functional_bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..daf35f0de5c56697f0380d1f32056a918bcba691 --- /dev/null +++ b/src/service/tests/qkd/test_functional_bootstrap.py @@ -0,0 +1,152 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time, json, socket, re +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +# Update the path to your QKD descriptor file +DESCRIPTOR_FILE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'descriptorQKD_links.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def load_descriptor_with_runtime_ip(descriptor_file_path): + """ + Load the descriptor file and replace placeholder IP with the machine's IP address. + """ + with open(descriptor_file_path, 'r') as descriptor_file: + descriptor = descriptor_file.read() + + # Get the current machine's IP address + try: + # Use socket to get the local IP address directly from the network interface + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + current_ip = s.getsockname()[0] + s.close() + except Exception as e: + raise Exception(f"Unable to get the IP address: {str(e)}") + + # Replace all occurrences of with the current IP + updated_descriptor = re.sub(r"", current_ip, descriptor) + + # Write updated descriptor back + with open(descriptor_file_path, 'w') as descriptor_file: + descriptor_file.write(updated_descriptor) + + return json.loads(updated_descriptor) + +def load_and_process_descriptor(context_client, device_client, descriptor_file_path): + """ + Function to load and process descriptor programmatically, similar to what WebUI does. + """ + print(f"Loading descriptor from file: {descriptor_file_path}") + try: + # Update the descriptor with the runtime IP address + descriptor = load_descriptor_with_runtime_ip(descriptor_file_path) + + # Initialize DescriptorLoader with the updated descriptor file + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file_path, context_client=context_client, device_client=device_client + ) + + # Process and validate the descriptor + print("Processing the descriptor...") + results = descriptor_loader.process() + print(f"Descriptor processing results: {results}") + + print("Checking descriptor load results...") + check_descriptor_load_results(results, descriptor_loader) + + print("Validating descriptor...") + descriptor_loader.validate() + print("Descriptor validated successfully.") + except Exception as e: + LOGGER.error(f"Failed to load and process descriptor: {e}") + raise e + +def test_qkd_scenario_bootstrap( + context_client: ContextClient, # pylint: disable=redefined-outer-name + device_client: DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the QKD scenario is correctly bootstrapped. + """ + print("Starting QKD scenario bootstrap test...") + + # Check if context_client and device_client are instantiated + if context_client is None: + print("Error: context_client is not instantiated!") + else: + print(f"context_client is instantiated: {context_client}") + + if device_client is None: + print("Error: device_client is not instantiated!") + else: + print(f"device_client is instantiated: {device_client}") + + # Validate empty scenario + print("Validating empty scenario...") + validate_empty_scenario(context_client) + + # Load the descriptor + load_and_process_descriptor(context_client, device_client, DESCRIPTOR_FILE_PATH) + +def test_qkd_devices_enabled( + context_client: ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the QKD devices are enabled. + """ + print("Starting QKD devices enabled test...") + + # Check if context_client is instantiated + if context_client is None: + print("Error: context_client is not instantiated!") + else: + print(f"context_client is instantiated: {context_client}") + + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + + while (num_devices != num_devices_enabled) and (num_retry < 10): + print(f"Attempt {num_retry + 1}: Checking device status...") + + time.sleep(1.0) # Add a delay to allow for device enablement + + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + print(f"Total devices found: {num_devices}") + + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status == DEVICE_OP_STATUS_ENABLED: + num_devices_enabled += 1 + + print(f"Devices enabled: {num_devices_enabled}/{num_devices}") + num_retry += 1 + + # Final check to ensure all devices are enabled + print(f"Final device status: {num_devices_enabled}/{num_devices} devices enabled.") + assert num_devices_enabled == num_devices + print("QKD devices enabled test completed.") \ No newline at end of file diff --git a/src/tests/.gitlab-ci.yml b/src/tests/.gitlab-ci.yml index dfccc4726b69c6346facb64c437364ff3b4b8aeb..4b7af45a042177ae70df84e710cb05cbcda79341 100644 --- a/src/tests/.gitlab-ci.yml +++ b/src/tests/.gitlab-ci.yml @@ -25,5 +25,7 @@ include: #- local: '/src/tests/ofc25-camara-e2e-controller/.gitlab-ci.yml' #- local: '/src/tests/ofc25/.gitlab-ci.yml' #- local: '/src/tests/ryu-openflow/.gitlab-ci.yml' + - local: '/src/tests/qkd_end2end/.gitlab-ci.yml' - local: '/src/tests/tools/mock_tfs_nbi_dependencies/.gitlab-ci.yml' + - local: '/src/tests/tools/mock_qkd_node/.gitlab-ci.yml' diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index aaab9f4f417cde748da234f2bba274f6e12b40b8..968b2333a45df79527fd41e122306fd31ccbde61 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ecoc22: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ecoc22: - timeout: 20m + timeout: 45m variables: TEST_NAME: 'ecoc22' stage: end2end_test @@ -45,13 +45,74 @@ end2end_test ecoc22: #needs: # - build ecoc22 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment @@ -62,6 +123,7 @@ end2end_test ecoc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -76,7 +138,7 @@ end2end_test ecoc22: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -84,11 +146,13 @@ end2end_test ecoc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server @@ -96,8 +160,18 @@ end2end_test ecoc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Clean up + - kubectl delete namespaces tfs || true + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force + + # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/tests/eucnc24/.gitlab-ci.yml b/src/tests/eucnc24/.gitlab-ci.yml index 72ef8ff8165c471e04d9ab8c4e2ffbb98227a929..70bdeba398c535cc0151a138683efe5892289397 100644 --- a/src/tests/eucnc24/.gitlab-ci.yml +++ b/src/tests/eucnc24/.gitlab-ci.yml @@ -37,7 +37,7 @@ build eucnc24: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test eucnc24: - timeout: 90m + timeout: 45m variables: TEST_NAME: 'eucnc24' stage: end2end_test @@ -45,16 +45,77 @@ end2end_test eucnc24: #needs: # - build eucnc24 before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f ${TEST_NAME} || true + # Cleanup old ContainerLab scenarios - containerlab destroy --all --cleanup || true + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Deploy ContainerLab Scenario @@ -83,7 +144,6 @@ end2end_test eucnc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" @@ -95,7 +155,7 @@ end2end_test eucnc24: - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/kafka.sh - - ./deploy/qdb.sh + #- ./deploy/qdb.sh - ./deploy/tfs.sh - ./deploy/show.sh @@ -171,7 +231,7 @@ end2end_test eucnc24: # Run end-to-end test: configure service IETF - > - docker run -t --rm --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-service-ietf-create.sh @@ -233,10 +293,8 @@ end2end_test eucnc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - # Destroy Scenario - - docker rm -f ${TEST_NAME} || true + # Clean up - RUNNER_PATH=`pwd` #- cd $PWD/src/tests/${TEST_NAME} - cd /tmp/clab/${TEST_NAME} @@ -244,6 +302,11 @@ end2end_test eucnc24: - sudo rm -rf clab-eucnc24/ .eucnc24.clab.yml.bak || true - cd $RUNNER_PATH - kubectl delete namespaces tfs || true + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 90adc40e988746ea08ed7cb96d58ff35e2abf1a6..70f13f7d56bd5b592980aa01989c559c1cb2e4ed 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ofc22: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ofc22: - timeout: 20m + timeout: 45m variables: TEST_NAME: 'ofc22' stage: end2end_test @@ -45,13 +45,74 @@ end2end_test ofc22: #needs: # - build ofc22 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: # Download Docker image to run the test - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Configure TeraFlowSDN deployment @@ -63,6 +124,7 @@ end2end_test ofc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -77,7 +139,7 @@ end2end_test ofc22: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS - while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -85,11 +147,13 @@ end2end_test ofc22: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + after_script: + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server @@ -99,8 +163,18 @@ end2end_test ofc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/monitoringservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice -c ztpservice - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Clean up + - kubectl delete namespaces tfs || true + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force + + # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml index 0a31dd21fb0d53ec31a24d9621241dbac03bd215..6579d7c2ea3446be3ebc85d2cb8152b221caf2d5 100644 --- a/src/tests/ofc24/.gitlab-ci.yml +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -37,7 +37,7 @@ build ofc24: # Deploy TeraFlowSDN and Execute end-2-end test end2end_test ofc24: - timeout: 90m + timeout: 45m variables: TEST_NAME: 'ofc24' stage: end2end_test @@ -45,9 +45,55 @@ end2end_test ofc24: #needs: # - build ofc24 before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - docker rm -f na-t1 na-t2 na-r1 na-r2 - - docker network rm -f na-br script: # Download Docker image to run the test @@ -57,6 +103,18 @@ end2end_test ofc24: # Check MicroK8s is ready - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done - kubectl get pods --all-namespaces # Deploy Optical Device Node Agents @@ -104,6 +162,7 @@ end2end_test ofc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" #- export TFS_SKIP_BUILD="YES" @@ -118,7 +177,7 @@ end2end_test ofc24: - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - + # Wait for Context to be subscribed to NATS #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -126,12 +185,19 @@ end2end_test ofc24: # Run end-to-end tests - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - > - docker run -t --name ${TEST_NAME} --network=host + docker run -t --rm --name ${TEST_NAME} --network=host --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" $CI_REGISTRY_IMAGE/${TEST_NAME}:latest after_script: + # Dump Optical Device Node Agents container status and logs + - docker ps -a + - docker logs na-t1 + - docker logs na-t2 + - docker logs na-r1 + - docker logs na-r2 + # Dump TeraFlowSDN component logs - source src/tests/${TEST_NAME}/deploy_specs.sh - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server @@ -140,18 +206,14 @@ end2end_test ofc24: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server - - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi - - # Dump Optical Device Node Agents container status and logs - - docker ps -a - - docker logs na-t1 - - docker logs na-t2 - - docker logs na-r1 - - docker logs na-r2 - # Destroy Optical Device Node Agents - - docker rm -f na-t1 na-t2 na-r1 na-r2 - - docker network rm -f na-br + # Clean up + - kubectl delete namespaces tfs || true + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force # Clean old docker images - docker images --filter="dangling=true" --quiet | xargs -r docker rmi diff --git a/src/tests/qkd_end2end/.gitignore b/src/tests/qkd_end2end/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..3ccc21c7db78aac26daa1f8c5ff8e1ffd3f35460 --- /dev/null +++ b/src/tests/qkd_end2end/.gitignore @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/.gitlab-ci.yml b/src/tests/qkd_end2end/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..62d97f8ca31d90fbea25a68d4e6a29767ead118c --- /dev/null +++ b/src/tests/qkd_end2end/.gitlab-ci.yml @@ -0,0 +1,348 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build qkd_end2end: + variables: + TEST_NAME: 'qkd_end2end' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test qkd_end2end: + timeout: 45m + variables: + TEST_NAME: 'qkd_end2end' + stage: end2end_test + # Disable to force running it after all other tasks + #needs: + # - build qkd_end2end + before_script: + # Do Docker cleanup + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker image prune --force + - docker network prune --force + - docker volume prune --all --force + - docker buildx prune --force + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Always delete Kubernetes namespaces + - export K8S_NAMESPACES=$(kubectl get namespace -o jsonpath='{.items[*].metadata.name}') + - echo "K8S_NAMESPACES=${K8S_NAMESPACES}" + + - export OLD_NATS_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^nats') + - echo "OLD_NATS_NAMESPACES=${OLD_NATS_NAMESPACES}" + - > + for ns in ${OLD_NATS_NAMESPACES}; do + if [[ "$ns" == nats* ]]; then + helm3 uninstall "$ns" -n "$ns" + fi + done + - export OLD_NAMESPACES=$(echo "${K8S_NAMESPACES}" | tr ' ' '\n' | grep -E '^(tfs|crdb|qdb|kafka|nats)') + - echo "OLD_NAMESPACES=${OLD_NAMESPACES}" + - kubectl delete namespace ${OLD_NAMESPACES} || true + + # Clean-up Kubernetes Failed pods + - > + kubectl get pods --all-namespaces --no-headers --field-selector=status.phase=Failed + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + xargs --no-run-if-empty --max-args=2 kubectl delete pod --namespace + + # Login Docker repository + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + + script: + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + - docker pull "${CI_REGISTRY_IMAGE}/mock-qkd-node:test" + + # Deploy scenario with mock QKD Nodes + - docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + - > + docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test + - > + docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test + - > + docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 + --volume "$PWD/src/tests/${TEST_NAME}/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" + ${CI_REGISTRY_IMAGE}/mock-qkd-node:test + + - echo "Waiting for QKD Nodes to initialize..." + - > + while ! docker logs qkd-node-01 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done + - > + while ! docker logs qkd-node-02 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done + - > + while ! docker logs qkd-node-03 2>&1 | grep -q "All log messages before absl::InitializeLog() is called are written to STDERR"; do + printf "%c" "." + sleep 1 + done + + # Dump logs of the QKD Nodes (script, before any configuration) + - docker logs qkd-node-01 + - docker logs qkd-node-02 + - docker logs qkd-node-03 + + # Dump configuration of the QKD Nodes (script, before any configuration) + - echo "[QKD-NODE-01] Config initial:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config initial:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config initial:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # Check MicroK8s is ready + - microk8s status --wait-ready + - LOOP_MAX_ATTEMPTS=10 + - LOOP_COUNTER=0 + - > + while ! kubectl get pods --all-namespaces &> /dev/null; do + printf "%c" "." + sleep 1 + LOOP_COUNTER=$((LOOP_COUNTER + 1)) + if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + echo "Max attempts reached, exiting the loop." + exit 1 + fi + done + - kubectl get pods --all-namespaces + + # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/qkd_appservice.yaml + + - source src/tests/${TEST_NAME}/deploy_specs.sh + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + + # Deploy TeraFlowSDN + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/kafka.sh + #- ./deploy/qdb.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + + ## Wait for Context to be subscribed to NATS + ## WARNING: this loop is infinite if there is no subscriber (such as monitoring). + ## Investigate if we can use a counter to limit the number of iterations. + ## For now, keep it commented out. + #- LOOP_MAX_ATTEMPTS=180 + #- LOOP_COUNTER=0 + #- > + # while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do + # echo "Attempt: $LOOP_COUNTER" + # kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1; + # sleep 1; + # LOOP_COUNTER=$((LOOP_COUNTER + 1)) + # if [ "$LOOP_COUNTER" -ge "$LOOP_MAX_ATTEMPTS" ]; then + # echo "Max attempts reached, exiting the loop." + # break + # fi + # done + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + + # Run end-to-end test: onboard scenario + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-01-onboarding.sh + + # Run end-to-end test: create QKD links + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-02-create-links.sh + + # Dump configuration of the QKD Nodes (script, after create QKD links) + - echo "[QKD-NODE-01] Config with links:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # TODO: check config of QKD Nodes with created links is correct + + # Run end-to-end test: create external app + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-03-create-external-app.sh + + # Dump configuration of the QKD Nodes (script, after create external app) + - echo "[QKD-NODE-01] Config with links and external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links and external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links and external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # TODO: check config of QKD Nodes with created links and external app is correct + + # Run end-to-end test: delete external app + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-04-delete-external-app.sh + + # Dump configuration of the QKD Nodes (script, after delete external app) + - echo "[QKD-NODE-01] Config with links and deleted external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with links and deleted external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with links and deleted external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # TODO: check config of QKD Nodes with created links and deleted external app is correct + + # Run end-to-end test: delete QKD links + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-05-delete-links.sh + + # Dump configuration of the QKD Nodes (script, after delete QKD links) + - echo "[QKD-NODE-01] Config with deleted links and deleted external app:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config with deleted links and deleted external app:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config with deleted links and deleted external app:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # TODO: check config of QKD Nodes with deleted links and deleted external app is correct + + # Run end-to-end test: cleanup scenario + - > + docker run -t --rm --name ${TEST_NAME} --network=host + --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + $CI_REGISTRY_IMAGE/${TEST_NAME}:latest /var/teraflow/run-06-cleanup.sh + + after_script: + # Dump logs of the QKD Nodes (after_script) + - docker ps -a + - docker logs qkd-node-01 + - docker logs qkd-node-02 + - docker logs qkd-node-03 + + # Dump configuration of the QKD Nodes (after_script) + - echo "[QKD-NODE-01] Config after_script:" + - curl "http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-02] Config after_script:" + - curl "http://172.254.250.102:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + - echo "[QKD-NODE-03] Config after_script:" + - curl "http://172.254.250.103:8080/restconf/data/etsi-qkd-sdn-node:" + - echo + + # Dump TeraFlowSDN component logs + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/qkd-appservice -c server + + # Clean up + - kubectl delete namespaces tfs || true + - docker ps --all --quiet | xargs --no-run-if-empty docker stop + - docker container prune --force + - docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + - docker network prune --force + - docker volume prune --all --force + + # Clean old docker images + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/qkd_end2end/Dockerfile b/src/tests/qkd_end2end/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..4486522b6d335d8ce0ef4cb84bb0a1e070636d3c --- /dev/null +++ b/src/tests/qkd_end2end/Dockerfile @@ -0,0 +1,88 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/qkd_end2end +WORKDIR /var/teraflow/tests/qkd_end2end +COPY src/tests/qkd_end2end/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/qkd_app/__init__.py qkd_app/__init__.py +COPY src/qkd_app/client/. qkd_app/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/vnt_manager/__init__.py vnt_manager/__init__.py +COPY src/vnt_manager/client/. vnt_manager/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/qkd_end2end/__init__.py ./tests/qkd_end2end/__init__.py +COPY src/tests/qkd_end2end/data/. ./tests/qkd_end2end/data/ +COPY src/tests/qkd_end2end/tests/. ./tests/qkd_end2end/tests/ +COPY src/tests/qkd_end2end/scripts/. ./ + +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install tree && \ + rm -rf /var/lib/apt/lists/* + +RUN tree -la /var/teraflow diff --git a/src/tests/qkd_end2end/README.md b/src/tests/qkd_end2end/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f00ef4ee17c042faaa921f5bd09a938de7a46166 --- /dev/null +++ b/src/tests/qkd_end2end/README.md @@ -0,0 +1,48 @@ +# QKD End-to-End Test + +## Emulated QKD Nodes +See `src/tests/tools/mock_qkd_node`. +Here we deploy 3 emulated QKD Nodes initialized with configurations `data/qkd-node-XX.json`. + +### (Re-)Deploy the 3 QKD Nodes +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/redeploy-qkd-nodes.sh +``` + +### Check their configuration +```bash +curl http://:/restconf/data/etsi-qkd-sdn-node: +``` + +### Update their configuration using root path +```bash +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://:/restconf/data/etsi-qkd-sdn-node: +``` + +### Update their configuration using sub-entity path +```bash +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://:/restconf/data/etsi-qkd-sdn-node:qkd_node +``` + +### Destroy scenario +```bash +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br +``` + +## TeraFlowSDN Deployment +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/redeploy-tfs.sh +``` + +### QKD Node Topology +The topology descriptor for the QKD nodes is: `data/tfs-01-topology.json` + +### Dump TFS component logs to files +```bash +cd ~/tfs-ctrl +./src/tests/qkd_end2end/dump_logs.sh +``` +Will create files `.log` diff --git a/src/tests/qkd_end2end/__init__.py b/src/tests/qkd_end2end/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ccc21c7db78aac26daa1f8c5ff8e1ffd3f35460 --- /dev/null +++ b/src/tests/qkd_end2end/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/data/qkd-node-01.json b/src/tests/qkd_end2end/data/qkd-node-01.json new file mode 100644 index 0000000000000000000000000000000000000000..39d518faa6c7c4c54c2fb4817b1772b175359594 --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-01.json @@ -0,0 +1,36 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000001-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "100", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "102", + "qkdi_att_point": { + "device": "QKD2", + "port": "201" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/data/qkd-node-02.json b/src/tests/qkd_end2end/data/qkd-node-02.json new file mode 100644 index 0000000000000000000000000000000000000000..2ffa647b15bb178233d22309cb476227059dfd2f --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-02.json @@ -0,0 +1,44 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000002-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "200", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "201", + "qkdi_att_point": { + "device": "QKD1", + "port": "102" + }, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "203", + "qkdi_att_point": { + "device": "QKD3", + "port": "302" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/data/qkd-node-03.json b/src/tests/qkd_end2end/data/qkd-node-03.json new file mode 100644 index 0000000000000000000000000000000000000000..276f2a2e0c0d52e3a97d293ed546b3984471bb1a --- /dev/null +++ b/src/tests/qkd_end2end/data/qkd-node-03.json @@ -0,0 +1,36 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000003-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": { + "qkd_app": [] + }, + "qkd_interfaces": { + "qkd_interface": [ + { + "qkdi_id": "300", + "qkdi_att_point": {}, + "qkdi_capabilities": {} + }, + { + "qkdi_id": "302", + "qkdi_att_point": { + "device": "QKD2", + "port": "203" + }, + "qkdi_capabilities": {} + } + ] + }, + "qkd_links": { + "qkd_link": [] + } + } + } +} diff --git a/src/tests/qkd_end2end/data/tfs-01-topology.json b/src/tests/qkd_end2end/data/tfs-01-topology.json new file mode 100644 index 0000000000000000000000000000000000000000..fe8aa367c42afd30f68a52f35b47b1af9549dd45 --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-01-topology.json @@ -0,0 +1,52 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "QKD1"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.101"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "QKD2"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.102"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "QKD3"}}, "device_type": "qkd-node", "device_drivers": ["DEVICEDRIVER_QKD"], + "device_config": {"config_rules": [ + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/address", "resource_value": "172.254.250.103"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/port", "resource_value": "8080"}}, + {"action": "CONFIGACTION_SET", "custom": {"resource_key": "_connect/settings", "resource_value": {"scheme": "http"}}} + ]} + } + ], + "links": [ + {"link_id": {"link_uuid": {"uuid": "QKD1/QKD2:201==QKD2/QKD1:102"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD2/QKD1:102==QKD1/QKD2:201"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}}, + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD2/QKD3:302==QKD3/QKD2:203"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} + ]}, + {"link_id": {"link_uuid": {"uuid": "QKD3/QKD2:203==QKD2/QKD3:302"}}, "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}} + ]} + ] +} diff --git a/src/tests/qkd_end2end/data/tfs-02-direct-link-qkd1-qkd2.json b/src/tests/qkd_end2end/data/tfs-02-direct-link-qkd1-qkd2.json new file mode 100644 index 0000000000000000000000000000000000000000..873ce6547f45600385860a50ef6172daad249200 --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-02-direct-link-qkd1-qkd2.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "direct-link-QKD1-QKD2"} + }, + "name": "direct-link-QKD1-QKD2", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD1:102"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/data/tfs-03-direct-link-qkd2-qkd3.json b/src/tests/qkd_end2end/data/tfs-03-direct-link-qkd2-qkd3.json new file mode 100644 index 0000000000000000000000000000000000000000..f9570ce8d9e9bf429d727cbc8deb8b9b8b56e4c3 --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-03-direct-link-qkd2-qkd3.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "direct-link-QKD2-QKD3"} + }, + "name": "direct-link-QKD2-QKD3", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD2"}}, "endpoint_uuid": {"uuid": "QKD3:302"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/data/tfs-04-virtual-link-qkd1-qkd3.json b/src/tests/qkd_end2end/data/tfs-04-virtual-link-qkd1-qkd3.json new file mode 100644 index 0000000000000000000000000000000000000000..206a8a279e1457fa2cb0194b32baed77177376cf --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-04-virtual-link-qkd1-qkd3.json @@ -0,0 +1,16 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "service_uuid": {"uuid": "virtual-link-QKD1-QKD3"} + }, + "name": "virtual-link-QKD1-QKD3", + "service_type": "SERVICETYPE_QKD", + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "QKD1"}}, "endpoint_uuid": {"uuid": "QKD2:201"}}, + {"device_id": {"device_uuid": {"uuid": "QKD3"}}, "endpoint_uuid": {"uuid": "QKD2:203"}} + ] + } + ] +} diff --git a/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json b/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json new file mode 100644 index 0000000000000000000000000000000000000000..95fcf24e096e351e66f7445d6bcdb8e74a1736f3 --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-05-app-1-qkd1-qkd3.json @@ -0,0 +1,9 @@ +{ + "app": { + "server_app_id": "1", + "client_app_id": [], + "app_status": "ON", + "local_qkdn_id": "00000001-0000-0000-0000-000000000000", + "backing_qkdl_id": ["00000003-0002-0000-0000-000000000000"] + } +} diff --git a/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json b/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json new file mode 100644 index 0000000000000000000000000000000000000000..1e76a317d2a8e86cdcc3c9dd151bda940d32130a --- /dev/null +++ b/src/tests/qkd_end2end/data/tfs-06-app-1-qkd3-qkd1.json @@ -0,0 +1,9 @@ +{ + "app": { + "server_app_id": "1", + "client_app_id": [], + "app_status": "ON", + "local_qkdn_id": "00000003-0000-0000-0000-000000000000", + "backing_qkdl_id": ["00000003-0002-0000-0000-000000000000"] + } +} diff --git a/src/tests/qkd_end2end/deploy_specs.sh b/src/tests/qkd_end2end/deploy_specs.sh new file mode 100755 index 0000000000000000000000000000000000000000..9a487d83afe47abce0f58cec1e04e8923770b213 --- /dev/null +++ b/src/tests/qkd_end2end/deploy_specs.sh @@ -0,0 +1,213 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +export TFS_COMPONENTS="context device pathcomp service nbi webui" + +# Uncomment to activate Monitoring (old) +#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate Monitoring Framework (new) +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" + +# Uncomment to activate QoS Profiles +#export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" + +# Uncomment to activate BGP-LS Speaker +#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" + +# Uncomment to activate Optical Controller +# To manage optical connections, "service" requires "opticalcontroller" to be deployed +# before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the +# "opticalcontroller" only if "service" is already in TFS_COMPONENTS, and re-export it. +#if [[ "$TFS_COMPONENTS" == *"service"* ]]; then +# BEFORE="${TFS_COMPONENTS% service*}" +# AFTER="${TFS_COMPONENTS#* service}" +# export TFS_COMPONENTS="${BEFORE} opticalcontroller service ${AFTER}" +#fi + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Uncomment to activate E2E Orchestrator +#export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" + +# Uncomment to activate VNT Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} vnt_manager" + +# Uncomment to activate DLT and Interdomain +#export TFS_COMPONENTS="${TFS_COMPONENTS} interdomain dlt" +#if [[ "$TFS_COMPONENTS" == *"dlt"* ]]; then +# export KEY_DIRECTORY_PATH="src/dlt/gateway/keys/priv_sk" +# export CERT_DIRECTORY_PATH="src/dlt/gateway/keys/cert.pem" +# export TLS_CERT_PATH="src/dlt/gateway/keys/ca.crt" +#fi + +# Uncomment to activate QKD App +# To manage QKD Apps, "service" requires "qkd_app" to be deployed +# before "service", thus we "hack" the TFS_COMPONENTS environment variable prepending the +# "qkd_app" only if "service" is already in TFS_COMPONENTS, and re-export it. +if [[ "$TFS_COMPONENTS" == *"service"* ]]; then + BEFORE="${TFS_COMPONENTS% service*}" + AFTER="${TFS_COMPONENTS#* service}" + export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" +fi + +# Uncomment to activate Load Generator +#export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" + + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Set NATS installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/nats.sh for additional details +export NATS_DEPLOY_MODE="single" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + +# Set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT="9092" + +# Set the flag to YES for redeploying of Apache Kafka +export KFK_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/src/tests/qkd_end2end/dump_logs.sh b/src/tests/qkd_end2end/dump_logs.sh new file mode 100755 index 0000000000000000000000000000000000000000..9abc47d7317fa4923311df0c543326c5db989b94 --- /dev/null +++ b/src/tests/qkd_end2end/dump_logs.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +kubectl --namespace tfs logs deployment/contextservice -c server > context.log +kubectl --namespace tfs logs deployment/deviceservice -c server > device.log +kubectl --namespace tfs logs deployment/pathcompservice -c frontend > pathcomp_front.log +kubectl --namespace tfs logs deployment/pathcompservice -c backend > pathcomp_back.log +kubectl --namespace tfs logs deployment/serviceservice -c server > service.log +kubectl --namespace tfs logs deployment/qkd-appservice -c server > qkd_app.log +kubectl --namespace tfs logs deployment/nbiservice -c server > nbi.log +kubectl --namespace tfs logs deployment/webuiservice -c server > webui.log diff --git a/src/tests/qkd_end2end/redeploy-all.sh b/src/tests/qkd_end2end/redeploy-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..b534e24658178d835efc9050fd59db3cf58aa7c7 --- /dev/null +++ b/src/tests/qkd_end2end/redeploy-all.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Cleanup +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br + +# Create Docker network +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + +# Create QKD Nodes +docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test + +# Dump QKD Node Docker containers +docker ps -a +echo + +# Wait till MicroK8s is stabilized... +microk8s status --wait-ready +echo + +source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh +./deploy/all.sh + +echo "Bye!" diff --git a/src/tests/qkd_end2end/redeploy-qkd-nodes.sh b/src/tests/qkd_end2end/redeploy-qkd-nodes.sh new file mode 100755 index 0000000000000000000000000000000000000000..b01e0ecb81ba0aa2f5690810c4a31ec1841bf955 --- /dev/null +++ b/src/tests/qkd_end2end/redeploy-qkd-nodes.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Cleanup +docker rm --force qkd-node-01 qkd-node-02 qkd-node-03 +docker network rm --force qkd-node-br + +# Create Docker network +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br + +# Create QKD Nodes +docker run --detach --name qkd-node-01 --network qkd-node-br --ip 172.254.250.101 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-01.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-02 --network qkd-node-br --ip 172.254.250.102 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-02.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test +docker run --detach --name qkd-node-03 --network qkd-node-br --ip 172.254.250.103 \ + --volume "$PWD/src/tests/qkd_end2end/data/qkd-node-03.json:/var/mock_qkd_node/startup.json" \ + mock-qkd-node:test + +# Dump QKD Node Docker containers +docker ps -a + +echo "Bye!" diff --git a/src/device/tests/qkd/unit/test_qkd_performance.py b/src/tests/qkd_end2end/redeploy-tfs.sh old mode 100644 new mode 100755 similarity index 54% rename from src/device/tests/qkd/unit/test_qkd_performance.py rename to src/tests/qkd_end2end/redeploy-tfs.sh index b0eafc31fed44e1ef8134e391429e0ab26ec6d1f..81b0ac5976390501496121b3502f1864f46b07f0 --- a/src/device/tests/qkd/unit/test_qkd_performance.py +++ b/src/tests/qkd_end2end/redeploy-tfs.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,21 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# tests/unit/test_qkd_performance.py -import pytest, time -from device.service.drivers.qkd.QKDDriver2 import QKDDriver - -MOCK_QKD_ADDRRESS = '127.0.0.1' -MOCK_PORT = 11111 - -def test_performance_under_load(): - driver = QKDDriver(address=MOCK_QKD_ADDRRESS, port=MOCK_PORT, username='user', password='pass') - driver.Connect() - - start_time = time.time() - for _ in range(1000): - driver.GetConfig(['/qkd_interfaces/qkd_interface']) - end_time = time.time() - - assert (end_time - start_time) < 60 +source ~/tfs-ctrl/src/tests/qkd_end2end/deploy_specs.sh +./deploy/all.sh diff --git a/src/tests/qkd_end2end/requirements.in b/src/tests/qkd_end2end/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..5c92783a232a5bbe18b4dd6d0e6735e3ce8414c2 --- /dev/null +++ b/src/tests/qkd_end2end/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +requests==2.27.* diff --git a/src/tests/qkd_end2end/scripts/run-01-onboarding.sh b/src/tests/qkd_end2end/scripts/run-01-onboarding.sh new file mode 100755 index 0000000000000000000000000000000000000000..df918620426d43ee38483f3298b2cdf73c707f16 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-01-onboarding.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_onboarding.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_01_onboarding.py diff --git a/src/tests/qkd_end2end/scripts/run-02-create-links.sh b/src/tests/qkd_end2end/scripts/run-02-create-links.sh new file mode 100755 index 0000000000000000000000000000000000000000..806d2ff1189e7bf3e8edd65ed30f89e349ab878d --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-02-create-links.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_create_links.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_02_create_links.py diff --git a/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh b/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh new file mode 100755 index 0000000000000000000000000000000000000000..45dcaf27e74a830756d7b1a897fb5f128b9d3103 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-03-create-external-app.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_ietf_create.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_03_create_external_app.py diff --git a/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh b/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh new file mode 100755 index 0000000000000000000000000000000000000000..e988042a55443751cf5c56fa8ea21740332559a4 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-04-delete-external-app.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_ietf_remove.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_04_delete_external_app.py diff --git a/src/tests/qkd_end2end/scripts/run-05-delete-links.sh b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh new file mode 100755 index 0000000000000000000000000000000000000000..8975759ce3f40c94a16e5439692c5d23b7667bc4 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-05-delete-links.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_service_tfs_remove.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_05_delete_links.py diff --git a/src/tests/qkd_end2end/scripts/run-06-cleanup.sh b/src/tests/qkd_end2end/scripts/run-06-cleanup.sh new file mode 100755 index 0000000000000000000000000000000000000000..991a5325eb365989e4735df8e93e4f5c77925a49 --- /dev/null +++ b/src/tests/qkd_end2end/scripts/run-06-cleanup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /var/teraflow/tfs_runtime_env_vars.sh +export PYTHONPATH=/var/teraflow +pytest --verbose --log-level=INFO \ + --junitxml=/opt/results/report_cleanup.xml \ + /var/teraflow/tests/qkd_end2end/tests/test_06_cleanup.py diff --git a/src/tests/qkd_end2end/tests/Fixtures.py b/src/tests/qkd_end2end/tests/Fixtures.py new file mode 100644 index 0000000000000000000000000000000000000000..788e29d5975d9f74c61fc2cdf3f73ebf6cfc346a --- /dev/null +++ b/src/tests/qkd_end2end/tests/Fixtures.py @@ -0,0 +1,43 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient + +@pytest.fixture(scope='session') +def context_client(): + _client = ContextClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def device_client(): + _client = DeviceClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def qkd_app_client(): + _client = QKDAppClient() + yield _client + _client.close() + +@pytest.fixture(scope='session') +def service_client(): + _client = ServiceClient() + yield _client + _client.close() diff --git a/src/tests/qkd_end2end/tests/Tools.py b/src/tests/qkd_end2end/tests/Tools.py new file mode 100644 index 0000000000000000000000000000000000000000..bbee845cd57f8dcb57e19f1f8ecc71940e99df30 --- /dev/null +++ b/src/tests/qkd_end2end/tests/Tools.py @@ -0,0 +1,109 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum, logging, requests +from typing import Any, Dict, List, Optional, Set, Union +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_http + +NBI_ADDRESS = get_service_host(ServiceNameEnum.NBI) +NBI_PORT = get_service_port_http(ServiceNameEnum.NBI) +NBI_USERNAME = 'admin' +NBI_PASSWORD = 'admin' +NBI_BASE_URL = '' + +class RestRequestMethod(enum.Enum): + GET = 'get' + POST = 'post' + PUT = 'put' + PATCH = 'patch' + DELETE = 'delete' + +EXPECTED_STATUS_CODES : Set[int] = { + requests.codes['OK' ], + requests.codes['CREATED' ], + requests.codes['ACCEPTED' ], + requests.codes['NO_CONTENT'], +} + +def do_rest_request( + method : RestRequestMethod, url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + request_url = 'http://{:s}:{:s}@{:s}:{:d}{:s}{:s}'.format( + NBI_USERNAME, NBI_PASSWORD, NBI_ADDRESS, NBI_PORT, str(NBI_BASE_URL), url + ) + + if logger is not None: + msg = 'Request: {:s} {:s}'.format(str(method.value).upper(), str(request_url)) + if body is not None: msg += ' body={:s}'.format(str(body)) + logger.warning(msg) + reply = requests.request(method.value, request_url, timeout=timeout, json=body, allow_redirects=allow_redirects) + if logger is not None: + logger.warning('Reply: {:s}'.format(str(reply.text))) + assert reply.status_code in expected_status_codes, 'Reply failed with status code {:d}'.format(reply.status_code) + + if reply.content and len(reply.content) > 0: return reply.json() + return None + +def do_rest_get_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.GET, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_post_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.POST, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_put_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PUT, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_patch_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PATCH, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_delete_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.DELETE, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) diff --git a/src/tests/qkd_end2end/tests/__init__.py b/src/tests/qkd_end2end/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ccc21c7db78aac26daa1f8c5ff8e1ffd3f35460 --- /dev/null +++ b/src/tests/qkd_end2end/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/qkd_end2end/tests/test_01_onboarding.py b/src/tests/qkd_end2end/tests/test_01_onboarding.py new file mode 100644 index 0000000000000000000000000000000000000000..e609856e72fdc8e391b15be5d99319aaece2b98f --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_01_onboarding.py @@ -0,0 +1,67 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-01-topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_onboarding( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 + assert num_devices_enabled == num_devices diff --git a/src/tests/qkd_end2end/tests/test_02_create_links.py b/src/tests/qkd_end2end/tests/test_02_create_links.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7587cf3bfd4026b14f7814f0e52b11df1fa019 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_02_create_links.py @@ -0,0 +1,235 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, Empty, ServiceStatusEnum, ServiceTypeEnum +from common.proto.qkd_app_pb2 import QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient +from .Fixtures import ( # pylint: disable=unused-import + context_client, device_client, service_client, qkd_app_client +) + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +def compose_path(file_name : str) -> str: + return os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', file_name) + + +def test_check_qkd_apps_before( + context_client : ContextClient, # pylint: disable=redefined-outer-name + qkd_app_client : QKDAppClient, # pylint: disable=redefined-outer-name +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 0 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 0 + +def test_create_direct_link_qkd1_qkd2( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-02-direct-link-qkd1-qkd2.json') + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 1 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_create_direct_link_qkd2_qkd3( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-03-direct-link-qkd2-qkd3.json') + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 2 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 2 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_create_virtual_link_qkd1_qkd3( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + descriptor_file = compose_path('tfs-04-virtual-link-qkd1-qkd3.json') + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=descriptor_file, context_client=context_client, + device_client=device_client, service_client=service_client + ) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 3 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format( + len(response.slices), grpc_message_to_json_string(response) + )) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + assert service.service_type == ServiceTypeEnum.SERVICETYPE_QKD + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), + grpc_message_to_json_string(response) + )) + assert len(response.connections) == 1 + +def test_check_qkd_apps_after( + context_client : ContextClient, # pylint: disable=redefined-outer-name + qkd_app_client : QKDAppClient, # pylint: disable=redefined-outer-name +): + response = context_client.ListDevices(Empty()) + LOGGER.warning('Devices[{:d}] = {:s}'.format( + len(response.devices), grpc_message_to_json_string(response) + )) + device_uuid_to_name = { + device.name : device.device_id.device_uuid.uuid + for device in response.devices + } + + device_qkd1_uuid = device_uuid_to_name.get('QKD1') + assert device_qkd1_uuid is not None + + device_qkd3_uuid = device_uuid_to_name.get('QKD3') + assert device_qkd3_uuid is not None + + pending_device_pairs = { + (device_qkd1_uuid, device_qkd3_uuid), + (device_qkd3_uuid, device_qkd1_uuid), + } + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + for app in response.apps: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + assert app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL + local_device_id = app.local_device_id.device_uuid.uuid + remote_device_id = app.remote_device_id.device_uuid.uuid + device_pair = (local_device_id, remote_device_id) + assert device_pair in pending_device_pairs + pending_device_pairs.remove(device_pair) + + assert len(pending_device_pairs) == 0 diff --git a/src/tests/qkd_end2end/tests/test_03_create_external_app.py b/src/tests/qkd_end2end/tests/test_03_create_external_app.py new file mode 100644 index 0000000000000000000000000000000000000000..f9efd8aba71057c7b99288afb9821dc11f0d74fa --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_03_create_external_app.py @@ -0,0 +1,123 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json, logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.proto.qkd_app_pb2 import QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from .Fixtures import context_client, qkd_app_client # pylint: disable=unused-import +from .Tools import do_rest_post_request + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +def compose_path(file_name : str) -> str: + return os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', file_name) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + for app in response.apps: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + assert app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL + + +# pylint: disable=redefined-outer-name +def test_create_external_app_qkd1_to_qkd3( + qkd_app_client : QKDAppClient, +): + request_file = compose_path('tfs-05-app-1-qkd1-qkd3.json') + + # Issue external QKD App creation request (QKD1-QKD3) + with open(request_file, 'r', encoding='UTF-8') as f: + req_data = json.load(f) + + URL = '/qkd_app/create_qkd_app' + do_rest_post_request(URL, body=req_data, logger=LOGGER, expected_status_codes={200}) + + # Verify QKD app was created + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 + + +# pylint: disable=redefined-outer-name +def test_create_external_app_qkd3_to_qkd1( + qkd_app_client : QKDAppClient, +): + request_file = compose_path('tfs-06-app-1-qkd3-qkd1.json') + + # Issue external QKD App creation request (QKD3-QKD1) + with open(request_file, 'r', encoding='UTF-8') as f: + req_data = json.load(f) + + URL = '/qkd_app/create_qkd_app' + do_rest_post_request(URL, body=req_data, logger=LOGGER, expected_status_codes={200}) + + # Verify no new QKD app was created + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 diff --git a/src/tests/qkd_end2end/tests/test_04_delete_external_app.py b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py new file mode 100644 index 0000000000000000000000000000000000000000..f352498a8e52bc8a321cae0f3a0b15986a50550f --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_04_delete_external_app.py @@ -0,0 +1,118 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.proto.qkd_app_pb2 import AppId, QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from .Fixtures import context_client, qkd_app_client # pylint: disable=unused-import + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are 3 services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 1 + + +# pylint: disable=redefined-outer-name +def test_delete_external_app( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 3 + + external_app_uuids : Set[str] = set() + for app in response.apps: + if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_CLIENT: continue + external_app_uuids.add(app.app_id.app_uuid.uuid) + + # Identify QKD ext app to delete + assert len(external_app_uuids) == 1 + external_app_uuid = set(external_app_uuids).pop() + + app_id = AppId() + app_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + app_id.app_uuid.uuid = external_app_uuid + + response = qkd_app_client.DeleteApp(app_id) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_after( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + num_external += 1 + assert num_internal == 2 + assert num_external == 0 diff --git a/src/tests/qkd_end2end/tests/test_05_delete_links.py b/src/tests/qkd_end2end/tests/test_05_delete_links.py new file mode 100644 index 0000000000000000000000000000000000000000..07a85963d11ba2e9d6f6b641942ff3d08258dad3 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_05_delete_links.py @@ -0,0 +1,141 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceId +from common.proto.qkd_app_pb2 import AppId, QKDAppStatusEnum, QKDAppTypesEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from qkd_app.client.QKDAppClient import QKDAppClient +from service.client.ServiceClient import ServiceClient +from .Fixtures import ( + # pylint: disable=unused-import + context_client, qkd_app_client, service_client +) + + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + + +# pylint: disable=redefined-outer-name +def test_check_qkd_apps_before( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + num_internal = 0 + num_external = 0 + for app in response.apps: + if app.app_type == QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_internal += 1 + elif app.app_type == QKDAppTypesEnum.QKDAPPTYPES_CLIENT: + assert app.app_status == QKDAppStatusEnum.QKDAPPSTATUS_ON + num_external += 1 + assert num_internal == 2 + assert num_external == 0 + + +# pylint: disable=redefined-outer-name +def test_delete_internal_apps( + context_client : ContextClient, + qkd_app_client : QKDAppClient, +): + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 2 + + # Identify internal QKD apps to delete + internal_app_uuids : Set[str] = set() + for app in response.apps: + if app.app_type != QKDAppTypesEnum.QKDAPPTYPES_INTERNAL: continue + internal_app_uuids.add(app.app_id.app_uuid.uuid) + + assert len(internal_app_uuids) == 2 + for app_uuid in internal_app_uuids: + app_id = AppId() + app_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + app_id.app_uuid.uuid = app_uuid + response = qkd_app_client.DeleteApp(app_id) + + response = qkd_app_client.ListApps(ADMIN_CONTEXT_ID) + LOGGER.warning('QKDApps[{:d}] = {:s}'.format( + len(response.apps), grpc_message_to_json_string(response) + )) + assert len(response.apps) == 0 + + +# pylint: disable=redefined-outer-name +def test_delete_services_associated_qkd_apps( + context_client : ContextClient, + service_client : ServiceClient, +): + # Check there are 3 services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 3 + + # Identify services for QKD links to delete + virtual_link_service_uuids : Set[str] = set() + direct_link_service_uuids : Set[str] = set() + for service in response.services: + if 'virtual' in str(service.name).lower(): + virtual_link_service_uuids.add(service.service_id.service_uuid.uuid) + if 'direct' in str(service.name).lower(): + direct_link_service_uuids.add(service.service_id.service_uuid.uuid) + + assert len(virtual_link_service_uuids) == 1 + assert len(direct_link_service_uuids ) == 2 + + # Delete the services for virtual links + for svc_uuid in virtual_link_service_uuids: + svc_id = ServiceId() + svc_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + svc_id.service_uuid.uuid = svc_uuid + response = service_client.DeleteService(svc_id) + + # Delete the services for direct links + for svc_uuid in direct_link_service_uuids: + svc_id = ServiceId() + svc_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME + svc_id.service_uuid.uuid = svc_uuid + response = service_client.DeleteService(svc_id) + + # Check there are no services + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format( + len(response.services), grpc_message_to_json_string(response) + )) + assert len(response.services) == 0 diff --git a/src/tests/qkd_end2end/tests/test_06_cleanup.py b/src/tests/qkd_end2end/tests/test_06_cleanup.py new file mode 100644 index 0000000000000000000000000000000000000000..e3c770e7a5e6be012e3311135b6346992eba9939 --- /dev/null +++ b/src/tests/qkd_end2end/tests/test_06_cleanup.py @@ -0,0 +1,44 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId +from common.tools.descriptor.Loader import DescriptorLoader, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'tfs-01-topology.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_cleanup( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) diff --git a/src/tests/tools/mock_qkd_node/.gitlab-ci.yml b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..59328359139e1ee943fadef81e438482a6df4137 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/.gitlab-ci.yml @@ -0,0 +1,39 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build mock_qkd_node: + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$CI_REGISTRY_IMAGE/mock-qkd-node:test" -f ./src/tests/tools/mock_qkd_node/Dockerfile ./src/tests/tools/mock_qkd_node + - docker push "$CI_REGISTRY_IMAGE/mock-qkd-node:test" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/src/tests/tools/mock_qkd_node/**/*.{py,in,yml,yaml,yang,sh,json} + - src/src/tests/tools/mock_qkd_node/Dockerfile + - src/device/**/*.{py,in,yml} + - src/device/Dockerfile + - src/device/tests/*.py + - src/qkd_app/**/*.{py,in,yml} + - src/qkd_app/Dockerfile + - src/qkd_app/tests/*.py + - .gitlab-ci.yml diff --git a/src/tests/tools/mock_qkd_node/Dockerfile b/src/tests/tools/mock_qkd_node/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5e39eb3c06ac5dac2379baf14bd99a8f500e2eb4 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Get Python dependencies +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install https://github.com/freeconf/lang/releases/download/v0.1.0-alpha/freeconf-0.1.0-py3-none-any.whl +RUN fc-lang-install -v + +# Create component sub-folders, and copy content +RUN mkdir -p /var/mock_qkd_node/ +WORKDIR /var/mock_qkd_node +COPY ./yang ./yang +COPY ./startup.json ./startup.json +COPY ./qkd_node.py ./qkd_node.py + +# Start the service +ENTRYPOINT ["python", "qkd_node.py"] diff --git a/src/tests/tools/mock_qkd_node/README.md b/src/tests/tools/mock_qkd_node/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7087fe061f2ca89879c819fede035763695943f1 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/README.md @@ -0,0 +1,34 @@ +# Mock QKD Node + +This Mock implements very basic support for the software-defined QKD node information models specified in ETSI GS QKD 015 V2.1.1. + +The aim of this mock is to enable testing the TFS QKD Framework with an emulated data plane. + + +## Build the Mock QKD Node Docker image +```bash +./build.sh +``` + +## Run the Mock QKD Node as a container: +```bash +docker network create --driver bridge --subnet=172.254.252.0/24 --gateway=172.254.252.254 tfs-qkd-net-mgmt + +docker run --name qkd-node-01 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.101 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-01.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test + +docker run --name qkd-node-02 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.102 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-02.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test + +docker run --name qkd-node-03 --detach --publish 80:80 \ + --network=tfs-qkd-net-mgmt --ip=172.254.252.103 \ + --env "DATA_FILE_PATH=/var/teraflow/mock-qkd-node/data/database.json" \ + --volume "$PWD/src/tests/mock-qkd-node/data/database-03.json:/var/teraflow/mock-qkd-node/data/database.json" \ + mock-qkd-node:test +``` diff --git a/src/tests/tools/mock_qkd_node/__init__.py b/src/tests/tools/mock_qkd_node/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ccc21c7db78aac26daa1f8c5ff8e1ffd3f35460 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/qkd/unit/test_mock_qkd_node.py b/src/tests/tools/mock_qkd_node/build.sh old mode 100644 new mode 100755 similarity index 50% rename from src/device/tests/qkd/unit/test_mock_qkd_node.py rename to src/tests/tools/mock_qkd_node/build.sh index 2ec060e873b311a38d1c0b91ae533d794bbddb69..e18dd98637481f7556a39c6dd95f0ff7846e4ed5 --- a/src/device/tests/qkd/unit/test_mock_qkd_node.py +++ b/src/tests/tools/mock_qkd_node/build.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,20 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest -import requests -from requests.exceptions import ConnectionError +# Make folder containing the script the root folder for its execution +cd $(dirname $0) -def test_mock_qkd_node_responses(): - response = requests.get('http://127.0.0.1:11111/restconf/data/etsi-qkd-sdn-node:qkd_node') - assert response.status_code == 200 - data = response.json() - assert 'qkd_node' in data - -def test_mock_node_failure_scenarios(): - try: - response = requests.get('http://127.0.0.1:12345/restconf/data/etsi-qkd-sdn-node:qkd_node') - except ConnectionError as e: - assert isinstance(e, ConnectionError) - else: - pytest.fail("ConnectionError not raised as expected") +docker buildx build -t mock-qkd-node:test -f Dockerfile . +#docker tag mock-qkd-node:test localhost:32000/tfs/mock-qkd-node:test +#docker push localhost:32000/tfs/mock-qkd-node:test diff --git a/src/tests/tools/mock_qkd_node/do_tests.sh b/src/tests/tools/mock_qkd_node/do_tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..02df6dd34485ba4e49bf5cb62e680b64a75842dc --- /dev/null +++ b/src/tests/tools/mock_qkd_node/do_tests.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Build the new Docker image +echo "Building..." +docker build -t mock-qkd-node:test -f Dockerfile . +echo + +# Run container +echo "Running..." +docker run -d --name qkd-node-01 -p 7777:7777 mock-qkd-node:test +echo + +# Give 5 seconds for container to start +echo "Waiting 3 seconds..." +for i in {1..3}; do + printf "%c" "." + sleep 1 +done +echo + +# Dumping containers before... +echo "Dumping containers before..." +docker ps -a +echo + +# Dumping logs before... +echo "Dumping logs before..." +docker logs qkd-node-01 +echo + +# Read data from startup +echo "Reading before..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Update data +echo "Updating 1..." +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo + +# Read data after 1 update +echo "Reading after 1..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Update data 2 +echo "Updating 2..." +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://localhost:7777/restconf/data/etsi-qkd-sdn-node:qkd_node +echo + +# Read data after 2 update +echo "Reading after 2..." +curl http://localhost:7777/restconf/data/etsi-qkd-sdn-node: +echo +echo + +# Dumping containers after... +echo "Dumping containers after..." +docker ps -a +echo + +# Dumping logs after... +echo "Dumping logs after..." +docker logs qkd-node-01 +echo + +# Destroy container +echo "Destroying..." +docker rm --force qkd-node-01 +echo + +echo "Bye!!" diff --git a/src/tests/tools/mock_qkd_node/qkd_node.py b/src/tests/tools/mock_qkd_node/qkd_node.py new file mode 100644 index 0000000000000000000000000000000000000000..e507a4619bd1c5c06d14f0605964f8093e29e11b --- /dev/null +++ b/src/tests/tools/mock_qkd_node/qkd_node.py @@ -0,0 +1,61 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from freeconf import restconf, source, device, parser, node, source, nodeutil +from threading import Event + +YANG_PATH = './yang' +YANG_MDL = 'etsi-qkd-sdn-node' +STARTUP_FILE = './startup.json' + +def main(): + # specify all the places where you store YANG files + yang_path = source.any( + source.path(YANG_PATH), # director to your local *.yang files + source.restconf_internal_ypath() # required for restconf protocol support + ) + + # load and validate your YANG file(s) + yang_module = parser.load_module_file(yang_path, YANG_MDL) + + # device hosts one or more management "modules" into a single instance that you + # want to export in the management interface + _device = device.Device(yang_path) + + # connect your application to your management implementation. + # there are endless ways to to build your management interface from code generation, + # to reflection and any combination there of. A lot more information in docs. + handler = nodeutil.Node(dict()) + + # connect parsed YANG to your management implementation. Browser is a powerful way + # to dynamically control your application can can be useful in unit tests or other contexts + # but here we construct it to serve our management API + browser = node.Browser(yang_module, handler) + + # register your app management browser in device. Device can hold any number of browsers + _device.add_browser(browser) + + # select RESTCONF as management protocol. gNMI is option as well or any custom or + # future protocols + restconf.Server(_device) + + # this will apply configuration including starting the RESTCONF web server + _device.apply_startup_config_file(STARTUP_FILE) + + # simple python trick to wait until ctrl-c shutdown + Event().wait() + +if __name__ == '__main__': + main() diff --git a/src/tests/tools/mock_qkd_nodes/start.sh b/src/tests/tools/mock_qkd_node/run.sh similarity index 57% rename from src/tests/tools/mock_qkd_nodes/start.sh rename to src/tests/tools/mock_qkd_node/run.sh index f0409747ca35d7b39c1bfa69a1f76df9cc2415ca..a017f9fed08975ef02c9bec9bbd7ab4ee60a7d24 100755 --- a/src/tests/tools/mock_qkd_nodes/start.sh +++ b/src/tests/tools/mock_qkd_node/run.sh @@ -13,30 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/bin/bash -cd "$(dirname "$0")" - -# Function to kill all background processes -killbg() { - for p in "${pids[@]}" ; do - kill "$p"; - done -} - -trap killbg EXIT -pids=() - -# Set FLASK_APP and run the Flask instances on different ports -export FLASK_APP=wsgi -flask run --host 0.0.0.0 --port 11111 & -pids+=($!) +# Cleanup +docker rm --force qkd-node +docker network rm --force qkd-node-br -flask run --host 0.0.0.0 --port 22222 & -pids+=($!) +# Create Docker network +docker network create --driver bridge --subnet=172.254.250.0/24 --gateway=172.254.250.254 qkd-node-br -flask run --host 0.0.0.0 --port 33333 & -pids+=($!) +# Create QKD Node +docker run --detach --name qkd-node --network qkd-node-br --ip 172.254.250.101 mock-qkd-node:test -# Wait for all background processes to finish -wait +# Dump QKD Node Docker containers +docker ps -a +echo "Bye!" diff --git a/src/tests/tools/mock_qkd_node/startup.json b/src/tests/tools/mock_qkd_node/startup.json new file mode 100644 index 0000000000000000000000000000000000000000..f1670f1ba2c8d2f337ea566a231b21f037596f47 --- /dev/null +++ b/src/tests/tools/mock_qkd_node/startup.json @@ -0,0 +1,16 @@ +{ + "fc-restconf": { + "web": { + "port": ":8080" + } + }, + "etsi-qkd-sdn-node": { + "qkd_node": { + "qkdn_id": "00000000-0000-0000-0000-000000000000", + "qkdn_capabilities": {}, + "qkd_applications": {}, + "qkd_interfaces": {}, + "qkd_links": {} + } + } +} diff --git a/src/tests/tools/mock_qkd_node/tests.sh b/src/tests/tools/mock_qkd_node/tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..81febc4ec00fa0068767867f07ae117ee63943fb --- /dev/null +++ b/src/tests/tools/mock_qkd_node/tests.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +echo "[QKD-NODE-01] Reading data on startup..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + +echo "[QKD-NODE-01] Updating location (from root)..." +curl -X PATCH -d '{"qkd_node":{"qkdn_location_id":"new-loc"}}' http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo + +echo "[QKD-NODE-01] Reading after update 1..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + +echo "[QKD-NODE-01] Updating location (from path)..." +curl -X PATCH -d '{"qkdn_location_id":"new-loc-2"}' http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node:qkd_node +echo + +echo "[QKD-NODE-01] Reading final value..." +curl http://172.254.250.101:8080/restconf/data/etsi-qkd-sdn-node: +echo +echo + diff --git a/src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-node-types.yang b/src/tests/tools/mock_qkd_node/yang/etsi-qkd-node-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-node-types.yang rename to src/tests/tools/mock_qkd_node/yang/etsi-qkd-node-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-sdn-node.yang b/src/tests/tools/mock_qkd_node/yang/etsi-qkd-sdn-node.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/etsi-qkd-sdn-node.yang rename to src/tests/tools/mock_qkd_node/yang/etsi-qkd-sdn-node.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/ietf-inet-types.yang b/src/tests/tools/mock_qkd_node/yang/ietf-inet-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/ietf-inet-types.yang rename to src/tests/tools/mock_qkd_node/yang/ietf-inet-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/yang/ietf-yang-types.yang b/src/tests/tools/mock_qkd_node/yang/ietf-yang-types.yang similarity index 100% rename from src/tests/tools/mock_qkd_nodes/yang/ietf-yang-types.yang rename to src/tests/tools/mock_qkd_node/yang/ietf-yang-types.yang diff --git a/src/tests/tools/mock_qkd_nodes/YangValidator.py b/src/tests/tools/mock_qkd_nodes/YangValidator.py deleted file mode 100644 index 4948239ed7430685699af2a7a4fafbcffd7dbb25..0000000000000000000000000000000000000000 --- a/src/tests/tools/mock_qkd_nodes/YangValidator.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import libyang, os -from typing import Dict, Optional - -YANG_DIR = os.path.join(os.path.dirname(__file__), 'yang') - -class YangValidator: - def __init__(self, main_module : str, dependency_modules : [str]) -> None: - self._yang_context = libyang.Context(YANG_DIR) - - self._yang_module = self._yang_context.load_module(main_module) - mods = [self._yang_context.load_module(mod) for mod in dependency_modules] + [self._yang_module] - - for mod in mods: - mod.feature_enable_all() - - - - def parse_to_dict(self, message : Dict) -> Dict: - dnode : Optional[libyang.DNode] = self._yang_module.parse_data_dict( - message, validate_present=True, validate=True, strict=True - ) - if dnode is None: raise Exception('Unable to parse Message({:s})'.format(str(message))) - message = dnode.print_dict() - dnode.free() - return message - - def destroy(self) -> None: - self._yang_context.destroy() diff --git a/src/tests/tools/mock_qkd_nodes/wsgi.py b/src/tests/tools/mock_qkd_nodes/wsgi.py deleted file mode 100644 index fde3c6cd024e96cb7693bb0f3036757b3177e353..0000000000000000000000000000000000000000 --- a/src/tests/tools/mock_qkd_nodes/wsgi.py +++ /dev/null @@ -1,368 +0,0 @@ -# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from flask import Flask, request -from YangValidator import YangValidator - -app = Flask(__name__) - - -yang_validator = YangValidator('etsi-qkd-sdn-node', ['etsi-qkd-node-types']) - - -nodes = { - '10.0.2.10:11111': {'node': { - 'qkdn_id': '00000001-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000001-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '100', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '101', - 'qkdi_att_point': { - 'device':'10.0.2.10', - 'port':'1001' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [ - - ] - } - }, - - '10.0.2.10:22222': {'node': { - 'qkdn_id': '00000002-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000002-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '200', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '201', - 'qkdi_att_point': { - 'device':'10.0.2.10', - 'port':'2001' - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '202', - 'qkdi_att_point': { - 'device':'10.0.2.10', - 'port':'2002' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [ - - ] - } - }, - - '10.0.2.10:33333': {'node': { - 'qkdn_id': '00000003-0000-0000-0000-000000000000', - }, - 'qkdn_capabilities': { - }, - 'qkd_applications': { - 'qkd_app': [ - { - 'app_id': '00000003-0001-0000-0000-000000000000', - 'client_app_id': [], - 'app_statistics': { - 'statistics': [] - }, - 'app_qos': { - }, - 'backing_qkdl_id': [] - } - ] - }, - 'qkd_interfaces': { - 'qkd_interface': [ - { - 'qkdi_id': '300', - 'qkdi_att_point': { - }, - 'qkdi_capabilities': { - } - }, - { - 'qkdi_id': '301', - 'qkdi_att_point': { - 'device':'10.0.2.10', - 'port':'3001' - }, - 'qkdi_capabilities': { - } - } - ] - }, - 'qkd_links': { - 'qkd_link': [ - - ] - } - } -} - - -def get_side_effect(url): - - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - ip_port, _, _, header, *steps = steps.split('/') - - header_splitted = header.split(':') - - module = header_splitted[0] - assert(module == 'etsi-qkd-sdn-node') - - tree = {'qkd_node': nodes[ip_port]['node'].copy()} - - if len(header_splitted) == 1 or not header_splitted[1]: - value = nodes[ip_port].copy() - value.pop('node') - tree['qkd_node'].update(value) - - return tree, tree - - root = header_splitted[1] - assert(root == 'qkd_node') - - if not steps: - return tree, tree - - - endpoint, *steps = steps - - value = nodes[ip_port][endpoint] - - if not steps: - return_value = {endpoint:value} - tree['qkd_node'].update(return_value) - - return return_value, tree - - - - ''' - element, *steps = steps - - container, key = element.split('=') - - # value = value[container][key] - - if not steps: - return_value['qkd_node'][endpoint] = [value] - return return_value - - ''' - raise Exception('Url too long') - - - -def edit(from_dict, to_dict, create): - for key, value in from_dict.items(): - if isinstance(value, dict): - if key not in to_dict and create: - to_dict[key] = {} - edit(from_dict[key], to_dict[key], create) - elif isinstance(value, list): - to_dict[key].extend(value) - else: - to_dict[key] = value - - - -def edit_side_effect(url, json, create): - steps = url.lstrip('https://').lstrip('http://').rstrip('/') - ip_port, _, _, header, *steps = steps.split('/') - - module, root = header.split(':') - - assert(module == 'etsi-qkd-sdn-node') - assert(root == 'qkd_node') - - if not steps: - edit(json, nodes[ip_port]['node']) - return - - endpoint, *steps = steps - - if not steps: - edit(json[endpoint], nodes[ip_port][endpoint], create) - return - - - ''' - element, *steps = steps - - container, key = element.split('=') - - if not steps: - if key not in nodes[ip_port][endpoint][container] and create: - nodes[ip_port][endpoint][container][key] = {} - - edit(json, nodes[ip_port][endpoint][container][key], create) - return 0 - ''' - - raise Exception('Url too long') - - - - - - -@app.get('/', defaults={'path': ''}) -@app.get("/") -@app.get('/') -def get(path): - msg, msg_validate = get_side_effect(request.base_url) - print(msg_validate) - yang_validator.parse_to_dict(msg_validate) - return msg - - -@app.post('/', defaults={'path': ''}) -@app.post("/") -@app.post('/') -def post(path): - success = True - reason = '' - try: - edit_side_effect(request.base_url, request.json, True) - except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} - - - -@app.route('/', defaults={'path': ''}, methods=['PUT', 'PATCH']) -@app.route("/", methods=['PUT', 'PATCH']) -@app.route('/', methods=['PUT', 'PATCH']) -def patch(path): - success = True - reason = '' - try: - edit_side_effect(request.base_url, request.json, False) - except Exception as e: - reason = str(e) - success = False - return {'success': success, 'reason': reason} - - - - - -# import json -# from mock import requests -# import pyangbind.lib.pybindJSON as enc -# from pyangbind.lib.serialise import pybindJSONDecoder as dec -# from yang.sbi.qkd.templates.etsi_qkd_sdn_node import etsi_qkd_sdn_node - -# module = etsi_qkd_sdn_node() -# url = 'https://1.1.1.1/restconf/data/etsi-qkd-sdn-node:' - -# # Get node all info -# z = requests.get(url).json() -# var = dec.load_json(z, None, None, obj=module) -# print(enc.dumps(var)) - - -# Reset module variable because it is already filled -# module = etsi_qkd_sdn_node() - -# # Get node basic info -# node = module.qkd_node -# z = requests.get(url + 'qkd_node').json() -# var = dec.load_json(z, None, None, obj=node) -# print(enc.dumps(var)) - - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var)) - -# # Edit app 0 -# app = apps.qkd_app['00000000-0001-0000-0000-000000000000'] -# app.client_app_id = 'id_0' -# requests.put(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000000', json=json.loads(enc.dumps(app))) - -# # Create app 1 -# app = apps.qkd_app.add('00000000-0001-0000-0000-000000000001') -# requests.post(url + 'qkd_node/qkd_applications/qkd_app=00000000-0001-0000-0000-000000000001', json=json.loads(enc.dumps(app))) - -# # Get all apps -# apps = node.qkd_applications -# z = requests.get(url + 'qkd_node/qkd_applications').json() -# var = dec.load_json(z, None, None, obj=apps) -# print(enc.dumps(var))