From 6c8f7184b707c289929314091094a7b9413b1824 Mon Sep 17 00:00:00 2001 From: Lluis Gifre <lluis.gifre@cttc.es> Date: Thu, 17 Feb 2022 09:11:35 +0000 Subject: [PATCH] Clarifications and corrections applied to release v1 --- .../add_license_header_to_files.sh | 0 .../report_coverage_all.sh | 0 .../report_coverage_common.sh | 0 .../report_coverage_compute.sh | 0 .../report_coverage_context.sh | 0 .../report_coverage_device.sh | 0 .../report_coverage_l3_attackmitigator.sh | 0 ...t_coverage_l3_centralizedattackdetector.sh | 0 ...t_coverage_l3_distributedattackdetector.sh | 0 .../report_coverage_service.sh | 0 .../run_tests_in_kubernetes.sh | 0 .../run_tests_locally.sh | 1 + src/tests/ofc22/README.md | 4 ++++ src/tests/ofc22/run_test_01_bootstrap.sh | 21 +++++++++++++------ src/tests/ofc22/run_test_02_create_service.sh | 15 +++++++------ src/tests/ofc22/run_test_03_delete_service.sh | 15 +++++++------ src/tests/ofc22/run_test_04_cleanup.sh | 15 +++++++------ src/tests/ofc22/show_logs_automation.sh | 3 ++- src/tests/ofc22/show_logs_compute.sh | 3 ++- src/tests/ofc22/show_logs_context.sh | 3 ++- src/tests/ofc22/show_logs_device.sh | 3 ++- src/tests/ofc22/show_logs_monitoring.sh | 3 ++- src/tests/ofc22/show_logs_service.sh | 3 ++- src/tests/ofc22/show_logs_webui.sh | 3 ++- src/webui/requirements.in | 6 +++--- 25 files changed, 64 insertions(+), 34 deletions(-) rename add_license_header_to_files.sh => scripts/add_license_header_to_files.sh (100%) rename report_coverage_all.sh => scripts/report_coverage_all.sh (100%) rename report_coverage_common.sh => scripts/report_coverage_common.sh (100%) rename report_coverage_compute.sh => scripts/report_coverage_compute.sh (100%) rename report_coverage_context.sh => scripts/report_coverage_context.sh (100%) rename report_coverage_device.sh => scripts/report_coverage_device.sh (100%) rename report_coverage_l3_attackmitigator.sh => scripts/report_coverage_l3_attackmitigator.sh (100%) rename report_coverage_l3_centralizedattackdetector.sh => scripts/report_coverage_l3_centralizedattackdetector.sh (100%) rename report_coverage_l3_distributedattackdetector.sh => scripts/report_coverage_l3_distributedattackdetector.sh (100%) rename report_coverage_service.sh => scripts/report_coverage_service.sh (100%) rename run_tests_in_kubernetes.sh => scripts/run_tests_in_kubernetes.sh (100%) rename run_tests_locally.sh => scripts/run_tests_locally.sh (98%) diff --git a/add_license_header_to_files.sh b/scripts/add_license_header_to_files.sh similarity index 100% rename from add_license_header_to_files.sh rename to scripts/add_license_header_to_files.sh diff --git a/report_coverage_all.sh b/scripts/report_coverage_all.sh similarity index 100% rename from report_coverage_all.sh rename to scripts/report_coverage_all.sh diff --git a/report_coverage_common.sh b/scripts/report_coverage_common.sh similarity index 100% rename from report_coverage_common.sh rename to scripts/report_coverage_common.sh diff --git a/report_coverage_compute.sh b/scripts/report_coverage_compute.sh similarity index 100% rename from report_coverage_compute.sh rename to scripts/report_coverage_compute.sh diff --git a/report_coverage_context.sh b/scripts/report_coverage_context.sh similarity index 100% rename from report_coverage_context.sh rename to scripts/report_coverage_context.sh diff --git a/report_coverage_device.sh b/scripts/report_coverage_device.sh similarity index 100% rename from report_coverage_device.sh rename to scripts/report_coverage_device.sh diff --git a/report_coverage_l3_attackmitigator.sh b/scripts/report_coverage_l3_attackmitigator.sh similarity index 100% rename from report_coverage_l3_attackmitigator.sh rename to scripts/report_coverage_l3_attackmitigator.sh diff --git a/report_coverage_l3_centralizedattackdetector.sh b/scripts/report_coverage_l3_centralizedattackdetector.sh similarity index 100% rename from report_coverage_l3_centralizedattackdetector.sh rename to scripts/report_coverage_l3_centralizedattackdetector.sh diff --git a/report_coverage_l3_distributedattackdetector.sh b/scripts/report_coverage_l3_distributedattackdetector.sh similarity index 100% rename from report_coverage_l3_distributedattackdetector.sh rename to scripts/report_coverage_l3_distributedattackdetector.sh diff --git a/report_coverage_service.sh b/scripts/report_coverage_service.sh similarity index 100% rename from report_coverage_service.sh rename to scripts/report_coverage_service.sh diff --git a/run_tests_in_kubernetes.sh b/scripts/run_tests_in_kubernetes.sh similarity index 100% rename from run_tests_in_kubernetes.sh rename to scripts/run_tests_in_kubernetes.sh diff --git a/run_tests_locally.sh b/scripts/run_tests_locally.sh similarity index 98% rename from run_tests_locally.sh rename to scripts/run_tests_locally.sh index 0f4fba608..1655d875e 100755 --- a/run_tests_locally.sh +++ b/scripts/run_tests_locally.sh @@ -25,6 +25,7 @@ cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/teraflow/controller+$PRO # Run unitary tests and analyze coverage of code at same time +# Set the name of the Kubernetes namespace and hostname to use. K8S_NAMESPACE="tf-dev" K8S_HOSTNAME="kubernetes-master" # Populate environment variables for context to use Redis in a development machine running Kubernetes diff --git a/src/tests/ofc22/README.md b/src/tests/ofc22/README.md index 3e50d2fe3..07fd4f72f 100644 --- a/src/tests/ofc22/README.md +++ b/src/tests/ofc22/README.md @@ -18,6 +18,10 @@ To run this functional test, it is assumed you have deployed a Kubernetes-based After installing Kubernetes, you can run it to deploy the appropriate components. Feel free to adapt it your particular case following the instructions described in [Wiki: Deploying a TeraFlow OS test instance](https://gitlab.com/teraflow-h2020/controller/-/wikis/Deploying-a-TeraFlow-OS-test-instance). +__Important:__ +- The `./ofc22/deploy_in_kubernetes.sh` assumes you have installed the appropriate development dependencies using the `install_development_dependencies.sh` script. +- Before running the scripts in this folder, remember to update the environment variable K8S_HOSTNAME to point to the Kubernetes node you will be using as described in [Wiki: Deploying a TeraFlow OS test instance](https://gitlab.com/teraflow-h2020/controller/-/wikis/Deploying-a-TeraFlow-OS-test-instance). + For your convenience, the configuration s sript `./ofc22/deploy_in_kubernetes.sh` has been already defined. The script will take some minutes to download the dependencies, build the micro-services, deploy them, and leave them ready for operation. The deployment will finish with a report of the items that have been created. ## Access to the WebUI and Dashboard diff --git a/src/tests/ofc22/run_test_01_bootstrap.sh b/src/tests/ofc22/run_test_01_bootstrap.sh index 9f61338a5..042e0a0a2 100755 --- a/src/tests/ofc22/run_test_01_bootstrap.sh +++ b/src/tests/ofc22/run_test_01_bootstrap.sh @@ -20,21 +20,30 @@ cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc COVERAGEFILE=$PROJECTDIR/coverage/.coverage -# Set the name of the Kubernetes namespace to deploy to. +# Configure the correct folder on the .coveragerc file +cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/teraflow/controller+$PROJECTDIR+g > $RCFILE + +# Destroy old coverage file +rm -f $COVERAGEFILE + +# Set the name of the Kubernetes namespace and hostname to use. K8S_NAMESPACE="ofc22" -K8S_NODENAME="kubernetes-master" +K8S_HOSTNAME="kubernetes-master" # Flush Context database kubectl --namespace $K8S_NAMESPACE exec -it deployment/contextservice --container redis -- redis-cli FLUSHALL -export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export CONTEXTSERVICE_SERVICE_PORT_GRPC=$(kubectl get service contextservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==1010)].nodePort}') -export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export DEVICESERVICE_SERVICE_PORT_GRPC=$(kubectl get service deviceservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==2020)].nodePort}') -export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export COMPUTESERVICE_SERVICE_PORT_HTTP=$(kubectl get service computeservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==8080)].nodePort}') -# Run integration tests and analyze coverage of code at same time +# Useful flags for pytest: +#-o log_cli=true -o log_file=device.log -o log_file_level=DEBUG + +# Run functional test and analyze coverage of code at same time coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ tests/ofc22/tests/test_functional_bootstrap.py diff --git a/src/tests/ofc22/run_test_02_create_service.sh b/src/tests/ofc22/run_test_02_create_service.sh index d2fbefb2e..b212b687c 100755 --- a/src/tests/ofc22/run_test_02_create_service.sh +++ b/src/tests/ofc22/run_test_02_create_service.sh @@ -20,18 +20,21 @@ cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc COVERAGEFILE=$PROJECTDIR/coverage/.coverage -# Set the name of the Kubernetes namespace to deploy to. +# Set the name of the Kubernetes namespace and hostname to use. K8S_NAMESPACE="ofc22" -K8S_NODENAME="kubernetes-master" +K8S_HOSTNAME="kubernetes-master" -export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export CONTEXTSERVICE_SERVICE_PORT_GRPC=$(kubectl get service contextservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==1010)].nodePort}') -export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export DEVICESERVICE_SERVICE_PORT_GRPC=$(kubectl get service deviceservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==2020)].nodePort}') -export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export COMPUTESERVICE_SERVICE_PORT_HTTP=$(kubectl get service computeservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==8080)].nodePort}') -# Run integration tests and analyze coverage of code at same time +# Useful flags for pytest: +#-o log_cli=true -o log_file=device.log -o log_file_level=DEBUG + +# Run functional test and analyze coverage of code at same time coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ tests/ofc22/tests/test_functional_create_service.py diff --git a/src/tests/ofc22/run_test_03_delete_service.sh b/src/tests/ofc22/run_test_03_delete_service.sh index cb7fbeb8d..d0c3127ad 100755 --- a/src/tests/ofc22/run_test_03_delete_service.sh +++ b/src/tests/ofc22/run_test_03_delete_service.sh @@ -20,18 +20,21 @@ cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc COVERAGEFILE=$PROJECTDIR/coverage/.coverage -# Set the name of the Kubernetes namespace to deploy to. +# Set the name of the Kubernetes namespace and hostname to use. K8S_NAMESPACE="ofc22" -K8S_NODENAME="kubernetes-master" +K8S_HOSTNAME="kubernetes-master" -export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export CONTEXTSERVICE_SERVICE_PORT_GRPC=$(kubectl get service contextservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==1010)].nodePort}') -export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export DEVICESERVICE_SERVICE_PORT_GRPC=$(kubectl get service deviceservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==2020)].nodePort}') -export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export COMPUTESERVICE_SERVICE_PORT_HTTP=$(kubectl get service computeservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==8080)].nodePort}') -# Run integration tests and analyze coverage of code at same time +# Useful flags for pytest: +#-o log_cli=true -o log_file=device.log -o log_file_level=DEBUG + +# Run functional test and analyze coverage of code at same time coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ tests/ofc22/tests/test_functional_delete_service.py diff --git a/src/tests/ofc22/run_test_04_cleanup.sh b/src/tests/ofc22/run_test_04_cleanup.sh index 6c48970d5..c31774523 100755 --- a/src/tests/ofc22/run_test_04_cleanup.sh +++ b/src/tests/ofc22/run_test_04_cleanup.sh @@ -20,18 +20,21 @@ cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc COVERAGEFILE=$PROJECTDIR/coverage/.coverage -# Set the name of the Kubernetes namespace to deploy to. +# Set the name of the Kubernetes namespace and hostname to use. K8S_NAMESPACE="ofc22" -K8S_NODENAME="kubernetes-master" +K8S_HOSTNAME="kubernetes-master" -export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export CONTEXTSERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export CONTEXTSERVICE_SERVICE_PORT_GRPC=$(kubectl get service contextservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==1010)].nodePort}') -export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export DEVICESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export DEVICESERVICE_SERVICE_PORT_GRPC=$(kubectl get service deviceservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==2020)].nodePort}') -export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_NODENAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') +export COMPUTESERVICE_SERVICE_HOST=$(kubectl get node $K8S_HOSTNAME -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}') export COMPUTESERVICE_SERVICE_PORT_HTTP=$(kubectl get service computeservice-public --namespace $K8S_NAMESPACE -o 'jsonpath={.spec.ports[?(@.port==8080)].nodePort}') -# Run integration tests and analyze coverage of code at same time +# Useful flags for pytest: +#-o log_cli=true -o log_file=device.log -o log_file_level=DEBUG + +# Run functional test and analyze coverage of code at same time coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ tests/ofc22/tests/test_functional_cleanup.py diff --git a/src/tests/ofc22/show_logs_automation.sh b/src/tests/ofc22/show_logs_automation.sh index 430681427..778cfaa94 100755 --- a/src/tests/ofc22/show_logs_automation.sh +++ b/src/tests/ofc22/show_logs_automation.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/automationservice +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/automationservice diff --git a/src/tests/ofc22/show_logs_compute.sh b/src/tests/ofc22/show_logs_compute.sh index fc2401d9a..cafde447a 100755 --- a/src/tests/ofc22/show_logs_compute.sh +++ b/src/tests/ofc22/show_logs_compute.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/computeservice +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/computeservice diff --git a/src/tests/ofc22/show_logs_context.sh b/src/tests/ofc22/show_logs_context.sh index e4dfdd212..6d5b77fa9 100755 --- a/src/tests/ofc22/show_logs_context.sh +++ b/src/tests/ofc22/show_logs_context.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/contextservice -c server +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/contextservice -c server diff --git a/src/tests/ofc22/show_logs_device.sh b/src/tests/ofc22/show_logs_device.sh index e150c0f51..9d976755a 100755 --- a/src/tests/ofc22/show_logs_device.sh +++ b/src/tests/ofc22/show_logs_device.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/deviceservice +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/deviceservice diff --git a/src/tests/ofc22/show_logs_monitoring.sh b/src/tests/ofc22/show_logs_monitoring.sh index 55d6316d6..3dd7522fa 100755 --- a/src/tests/ofc22/show_logs_monitoring.sh +++ b/src/tests/ofc22/show_logs_monitoring.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/monitoringservice -c server +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/monitoringservice -c server diff --git a/src/tests/ofc22/show_logs_service.sh b/src/tests/ofc22/show_logs_service.sh index 0659cbb04..2589a3cfe 100755 --- a/src/tests/ofc22/show_logs_service.sh +++ b/src/tests/ofc22/show_logs_service.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/serviceservice +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/serviceservice diff --git a/src/tests/ofc22/show_logs_webui.sh b/src/tests/ofc22/show_logs_webui.sh index 2f506d334..ecf4f3f6f 100755 --- a/src/tests/ofc22/show_logs_webui.sh +++ b/src/tests/ofc22/show_logs_webui.sh @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -kubectl --namespace ofc22 logs deployment/webuiservice -c server +K8S_NAMESPACE="ofc22" +kubectl --namespace $K8S_NAMESPACE logs deployment/webuiservice -c server diff --git a/src/webui/requirements.in b/src/webui/requirements.in index 534f42115..c57794403 100644 --- a/src/webui/requirements.in +++ b/src/webui/requirements.in @@ -1,11 +1,11 @@ -Flask==2.0.1 +Flask==2.0.2 Flask-WTF==1.0.0 flask-healthz==0.0.3 flask-unittest==0.1.2 grpcio==1.43.0 grpcio-health-checking==1.43.0 -protobuf==3.17.3 -prometheus-client==0.11.0 +protobuf==3.19.3 +prometheus-client==0.13.0 pytest==6.2.5 pytest-benchmark==3.4.1 lorem-text==2.1 -- GitLab