From 0a67951cfdbf6c6d853a203e7f16882faf90f899 Mon Sep 17 00:00:00 2001 From: mansoca Date: Fri, 18 Oct 2024 14:05:43 +0000 Subject: [PATCH 1/6] alpha --- src/tests/ecoc24/.gitlab-ci.yml | 124 +++++++++++++++++++++++++++ src/tests/ecoc24/Dockerfile | 102 ++++++++++++++++++++++ src/tests/ecoc24/deploy_specs_e2e.sh | 2 +- src/tests/ecoc24/deploy_specs_ip.sh | 2 +- src/tests/ecoc24/deploy_specs_opt.sh | 2 +- src/tests/ecoc24/tests/__init__.py | 14 +++ src/tests/ofc24/Dockerfile | 31 +++---- 7 files changed, 259 insertions(+), 18 deletions(-) create mode 100644 src/tests/ecoc24/.gitlab-ci.yml create mode 100644 src/tests/ecoc24/Dockerfile create mode 100644 src/tests/ecoc24/tests/__init__.py diff --git a/src/tests/ecoc24/.gitlab-ci.yml b/src/tests/ecoc24/.gitlab-ci.yml new file mode 100644 index 000000000..3fb66f399 --- /dev/null +++ b/src/tests/ecoc24/.gitlab-ci.yml @@ -0,0 +1,124 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build ecoc24: + variables: + TEST_NAME: 'ecoc24' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test ecoc24: + variables: + TEST_NAME: 'ecoc24' + stage: end2end_test + # Disable to force running it after all other tasks + #needs: + # - build ecoc24 + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker network rm -f na-br + + script: + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + + + # Check MicroK8s is ready + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + + # Deploy Optical Device Node Agents + - > + docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254 + --ip-range=172.254.253.0/24 na-br + + + # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + + # Deploy TeraFlowSDN + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/qdb.sh + - ./deploy/expose_dashboard.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + + # Wait for Context to be subscribed to NATS + #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + + # Run end-to-end tests + #- if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + #- > + # docker run -t --name ${TEST_NAME} --network=host + # --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + # --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + # $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + + after_script: + # Dump TeraFlowSDN component logs + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Dump Optical Device Node Agents container status and logs + - docker ps -a + + # Clean old docker images + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile new file mode 100644 index 000000000..734cf6233 --- /dev/null +++ b/src/tests/ecoc24/Dockerfile @@ -0,0 +1,102 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/ofc24 +WORKDIR /var/teraflow/tests/ofc24 +COPY src/tests/ofc24/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/ofc24/__init__.py ./tests/ofc24/__init__.py +COPY src/tests/ofc24/descriptors/topology.json ./tests/ofc24/descriptors/topology.json +COPY src/tests/ofc24/descriptors/service-unidir.json ./tests/ofc24/descriptors/service-unidir.json +COPY src/tests/ofc24/descriptors/service-bidir.json ./tests/ofc24/descriptors/service-bidir.json +COPY src/tests/ofc24/tests/. ./tests/ofc24/tests/ + +RUN tee ./run_tests.sh < Date: Fri, 18 Oct 2024 14:05:43 +0000 Subject: [PATCH 2/6] alpha --- src/tests/ecoc24/.gitlab-ci.yml | 124 +++++++++++++++++++++++++++ src/tests/ecoc24/Dockerfile | 102 ++++++++++++++++++++++ src/tests/ecoc24/deploy_specs_e2e.sh | 2 +- src/tests/ecoc24/deploy_specs_ip.sh | 2 +- src/tests/ecoc24/deploy_specs_opt.sh | 2 +- src/tests/ecoc24/tests/__init__.py | 14 +++ src/tests/ofc24/Dockerfile | 35 ++++---- 7 files changed, 261 insertions(+), 20 deletions(-) create mode 100644 src/tests/ecoc24/.gitlab-ci.yml create mode 100644 src/tests/ecoc24/Dockerfile create mode 100644 src/tests/ecoc24/tests/__init__.py diff --git a/src/tests/ecoc24/.gitlab-ci.yml b/src/tests/ecoc24/.gitlab-ci.yml new file mode 100644 index 000000000..3fb66f399 --- /dev/null +++ b/src/tests/ecoc24/.gitlab-ci.yml @@ -0,0 +1,124 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build ecoc24: + variables: + TEST_NAME: 'ecoc24' + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${TEST_NAME}:latest" -f ./src/tests/${TEST_NAME}/Dockerfile . + - docker tag "${TEST_NAME}:latest" "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + - docker push "$CI_REGISTRY_IMAGE/${TEST_NAME}:latest" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/tests/${TEST_NAME}/**/*.{py,in,sh,yml} + - src/tests/${TEST_NAME}/Dockerfile + - .gitlab-ci.yml + +# Deploy TeraFlowSDN and Execute end-2-end test +end2end_test ecoc24: + variables: + TEST_NAME: 'ecoc24' + stage: end2end_test + # Disable to force running it after all other tasks + #needs: + # - build ecoc24 + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker network rm -f na-br + + script: + # Download Docker image to run the test + - docker pull "${CI_REGISTRY_IMAGE}/${TEST_NAME}:latest" + + + # Check MicroK8s is ready + - microk8s status --wait-ready + - kubectl get pods --all-namespaces + + # Deploy Optical Device Node Agents + - > + docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254 + --ip-range=172.254.253.0/24 na-br + + + # Configure TeraFlowSDN deployment + # Uncomment if DEBUG log level is needed for the components + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/contextservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml + - source src/tests/${TEST_NAME}/deploy_specs.sh + #- export TFS_REGISTRY_IMAGES="${CI_REGISTRY_IMAGE}" + #- export TFS_SKIP_BUILD="YES" + #- export TFS_IMAGE_TAG="latest" + #- echo "TFS_REGISTRY_IMAGES=${CI_REGISTRY_IMAGE}" + + # Deploy TeraFlowSDN + - ./deploy/crdb.sh + - ./deploy/nats.sh + - ./deploy/qdb.sh + - ./deploy/expose_dashboard.sh + - ./deploy/tfs.sh + - ./deploy/show.sh + + # Wait for Context to be subscribed to NATS + #- while ! kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server 2>&1 | grep -q 'Subscriber is Ready? True'; do sleep 1; done + #- kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + + # Run end-to-end tests + #- if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + #- > + # docker run -t --name ${TEST_NAME} --network=host + # --volume "$PWD/tfs_runtime_env_vars.sh:/var/teraflow/tfs_runtime_env_vars.sh" + # --volume "$PWD/src/tests/${TEST_NAME}:/opt/results" + # $CI_REGISTRY_IMAGE/${TEST_NAME}:latest + + after_script: + # Dump TeraFlowSDN component logs + - source src/tests/${TEST_NAME}/deploy_specs.sh + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/contextservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server + - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/opticalcontrollerservice -c server + - if docker ps -a | grep ${TEST_NAME}; then docker rm -f ${TEST_NAME}; fi + + # Dump Optical Device Node Agents container status and logs + - docker ps -a + + # Clean old docker images + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + + #coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + artifacts: + when: always + reports: + junit: ./src/tests/${TEST_NAME}/report_*.xml diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile new file mode 100644 index 000000000..734cf6233 --- /dev/null +++ b/src/tests/ecoc24/Dockerfile @@ -0,0 +1,102 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/tests/ofc24 +WORKDIR /var/teraflow/tests/ofc24 +COPY src/tests/ofc24/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/__init__.py ./__init__.py +COPY src/common/*.py ./common/ +COPY src/common/tests/. ./common/tests/ +COPY src/common/tools/. ./common/tools/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ +COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py +COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ +COPY src/slice/__init__.py slice/__init__.py +COPY src/slice/client/. slice/client/ +COPY src/tests/*.py ./tests/ +COPY src/tests/ofc24/__init__.py ./tests/ofc24/__init__.py +COPY src/tests/ofc24/descriptors/topology.json ./tests/ofc24/descriptors/topology.json +COPY src/tests/ofc24/descriptors/service-unidir.json ./tests/ofc24/descriptors/service-unidir.json +COPY src/tests/ofc24/descriptors/service-bidir.json ./tests/ofc24/descriptors/service-bidir.json +COPY src/tests/ofc24/tests/. ./tests/ofc24/tests/ + +RUN tee ./run_tests.sh < Date: Tue, 5 Nov 2024 11:18:18 +0000 Subject: [PATCH 3/6] deploy --- manifests/e2e_orchestratorservice.yaml | 10 ++- manifests/nbiservice.yaml | 12 ++-- manifests/serviceservice.yaml | 2 +- .../service/drivers/oc_driver/OCDriver.py | 2 +- src/e2e_orchestrator/requirements.in | 1 + .../E2EOrchestratorServiceServicerImpl.py | 39 +++++++++-- .../service/context_subscription/__init__.py | 6 +- .../nbi_plugins/tfs_api/Resources.py | 5 ++ .../nbi_plugins/tfs_api/__init__.py | 53 ++++++++------- src/tests/ecoc24/.gitlab-ci.yml | 31 +++++---- src/tests/ecoc24/deploy_e2e.sh | 5 ++ src/tests/ecoc24/deploy_ip.sh | 5 ++ src/tests/ecoc24/deploy_opt.sh | 6 ++ src/tests/ecoc24/deploy_specs_e2e.sh | 5 +- src/tests/ecoc24/deploy_specs_ip.sh | 4 +- src/tests/ecoc24/deploy_specs_opt.sh | 4 +- ...{link_mapping.json => descriptor_e2e.json} | 3 +- .../ecoc24/descriptors/descriptor_ip.json | 3 +- .../ecoc24/descriptors/descriptor_opt.json | 3 +- .../tests/test_functional_bootstrap_e2e.py | 67 +++++++++++++++++++ ...rap.py => test_functional_bootstrap_ip.py} | 0 .../tests/test_functional_bootstrap_opt.py | 67 +++++++++++++++++++ .../service/VNTManagerServiceServicerImpl.py | 4 +- 23 files changed, 268 insertions(+), 69 deletions(-) rename src/tests/ecoc24/descriptors/{link_mapping.json => descriptor_e2e.json} (99%) create mode 100644 src/tests/ecoc24/tests/test_functional_bootstrap_e2e.py rename src/tests/ecoc24/tests/{test_functional_bootstrap.py => test_functional_bootstrap_ip.py} (100%) create mode 100644 src/tests/ecoc24/tests/test_functional_bootstrap_opt.py diff --git a/manifests/e2e_orchestratorservice.yaml b/manifests/e2e_orchestratorservice.yaml index 893f3464f..21e5d49cb 100644 --- a/manifests/e2e_orchestratorservice.yaml +++ b/manifests/e2e_orchestratorservice.yaml @@ -37,7 +37,7 @@ spec: ports: - containerPort: 10050 - containerPort: 9192 - - containerPort: 8761 + - containerPort: 8762 env: - name: LOG_LEVEL value: "INFO" @@ -49,6 +49,10 @@ spec: value: "e2e-orchestratorservice.tfs-e2e.svc.cluster.local" - name: WS_E2E_PORT value: "8762" + - name: EXT_CONTROLLER1_ADD + value: "10.1.1.96" + - name: EXT_CONTROLLER1_PORT + value: "8003" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:10050"] @@ -81,5 +85,5 @@ spec: port: 9192 targetPort: 9192 - name: ws - port: 8761 - targetPort: 8761 + port: 8762 + targetPort: 8762 diff --git a/manifests/nbiservice.yaml b/manifests/nbiservice.yaml index 72cfde514..ac9c9bec5 100644 --- a/manifests/nbiservice.yaml +++ b/manifests/nbiservice.yaml @@ -38,14 +38,14 @@ spec: - containerPort: 8080 - containerPort: 9090 - containerPort: 9192 - - containerPort: 8762 + - containerPort: 8761 env: - name: LOG_LEVEL value: "INFO" - name: IETF_NETWORK_RENDERER value: "LIBYANG" - - name: WS_E2E_PORT - value: "8762" + - name: WS_IP_PORT + value: "8761" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:9090"] @@ -83,7 +83,7 @@ spec: protocol: TCP port: 9192 targetPort: 9192 - - name: ws + - name: websocket protocol: TCP - port: 8762 - targetPort: 8762 + port: 8761 + targetPort: 8761 diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 1dd383d61..bcfb47ee3 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:3030"] diff --git a/src/device/service/drivers/oc_driver/OCDriver.py b/src/device/service/drivers/oc_driver/OCDriver.py index 6351f26e5..087e58b76 100644 --- a/src/device/service/drivers/oc_driver/OCDriver.py +++ b/src/device/service/drivers/oc_driver/OCDriver.py @@ -300,7 +300,7 @@ class OCDriver(_Driver): self.__netconf_handler, self.__logger, resources, conditions, target='candidate', commit_per_rule=self.__netconf_handler.commit_per_rule ) - else: + else: results = edit_config( self.__netconf_handler, self.__logger, resources, conditions=conditions ) diff --git a/src/e2e_orchestrator/requirements.in b/src/e2e_orchestrator/requirements.in index 5732b1bf0..7ab2d5ded 100644 --- a/src/e2e_orchestrator/requirements.in +++ b/src/e2e_orchestrator/requirements.in @@ -14,3 +14,4 @@ networkx websockets==12.0 +requests==2.27.* diff --git a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py index 4fc0ea3ba..52b5a21f8 100644 --- a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py +++ b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py @@ -13,6 +13,7 @@ # limitations under the License. import copy +import requests from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply from common.proto.context_pb2 import ( @@ -33,7 +34,7 @@ import networkx as nx from threading import Thread from websockets.sync.client import connect from websockets.sync.server import serve -from common.Constants import DEFAULT_CONTEXT_NAME +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME LOGGER = logging.getLogger(__name__) @@ -98,7 +99,6 @@ class SubscriptionServer(Thread): def _event_received(self, connection): - LOGGER.info("EVENT received!") for message in connection: message_json = json.loads(message) # LOGGER.info("message_json: {}".format(message_json)) @@ -142,7 +142,6 @@ class SubscriptionServer(Thread): connection.send(grpc_message_to_json_string(link)) # Link removal elif 'link_uuid' in message_json: - LOGGER.info('REMOVING VIRTUAL LINK') link_id = LinkId(**message_json) service_id = ServiceId() @@ -153,7 +152,6 @@ class SubscriptionServer(Thread): context_client.RemoveLink(link_id) # Topology received else: - LOGGER.info('TOPOLOGY') topology_details = TopologyDetails(**message_json) context = Context() @@ -181,10 +179,41 @@ class E2EOrchestratorServiceServicerImpl(E2EOrchestratorServiceServicer): sub_server = SubscriptionServer() sub_server.start() LOGGER.debug("Servicer Created") - + self.retrieve_external_topologies() except Exception as ex: LOGGER.info("Exception!: {}".format(ex)) + def retrieve_external_topologies(self): + i = 1 + while True: + try: + LOGGER.info(f'Retrieving external controller #{i}') + ADD = str(get_setting(f'EXT_CONTROLLER{i}_ADD')) + PORT = str(get_setting(f'EXT_CONTROLLER{i}_PORT')) + except Exception as e: + break + try: + url = f'http://{ADD}:{PORT}/tfs-api/context/{DEFAULT_CONTEXT_NAME}/topology_details/{DEFAULT_TOPOLOGY_NAME}' + topo = requests.get(url).json() + except Exception as e: + LOGGER.info(f'Exception retrieven topology from external controler #{i}: {e}') + topology_details = TopologyDetails(**topo) + context = Context() + context.context_id.context_uuid.uuid = topology_details.topology_id.context_id.context_uuid.uuid + context_client.SetContext(context) + + topology = Topology() + topology.topology_id.context_id.CopyFrom(context.context_id) + topology.topology_id.topology_uuid.uuid = topology_details.topology_id.topology_uuid.uuid + context_client.SetTopology(topology) + + for device in topology_details.devices: + context_client.SetDevice(device) + + for link in topology_details.links: + context_client.SetLink(link) + + i+=1 @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) diff --git a/src/nbi/service/context_subscription/__init__.py b/src/nbi/service/context_subscription/__init__.py index d2ae85070..e33728ece 100644 --- a/src/nbi/service/context_subscription/__init__.py +++ b/src/nbi/service/context_subscription/__init__.py @@ -34,14 +34,14 @@ vnt_manager_client: VNTManagerClient = VNTManagerClient() context_client: ContextClient = ContextClient() ALL_HOSTS = "0.0.0.0" -WS_E2E_PORT = int(get_setting('WS_E2E_PORT', default='8762')) +WS_IP_PORT = int(get_setting('WS_IP_PORT', default='8761')) LOGGER = logging.getLogger(__name__) def register_context_subscription(): - with serve(subcript_to_vnt_manager, ALL_HOSTS, WS_E2E_PORT, logger=LOGGER) as server: - LOGGER.info("Running subscription server...: {}:{}".format(ALL_HOSTS, str(WS_E2E_PORT))) + with serve(subcript_to_vnt_manager, ALL_HOSTS, WS_IP_PORT, logger=LOGGER) as server: + LOGGER.info("Running subscription server...: {}:{}".format(ALL_HOSTS, str(WS_IP_PORT))) server.serve_forever() LOGGER.info("Exiting subscription server...") diff --git a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py index 28f94887a..58cf9aa01 100644 --- a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py @@ -177,6 +177,11 @@ class Topology(_Resource): def delete(self, context_uuid : str, topology_uuid : str): return format_grpc_to_json(self.context_client.RemoveTopology(grpc_topology_id(context_uuid, topology_uuid))) +class TopologyDetails(_Resource): + def get(self, context_uuid : str, topology_uuid : str): + return format_grpc_to_json(self.context_client.GetTopologyDetails(grpc_topology_id( + context_uuid, topology_uuid))) + class ServiceIds(_Resource): def get(self, context_uuid : str): return format_grpc_to_json(self.context_client.ListServiceIds(grpc_context_id(context_uuid))) diff --git a/src/nbi/service/rest_server/nbi_plugins/tfs_api/__init__.py b/src/nbi/service/rest_server/nbi_plugins/tfs_api/__init__.py index 41e8ff1ea..984e0442a 100644 --- a/src/nbi/service/rest_server/nbi_plugins/tfs_api/__init__.py +++ b/src/nbi/service/rest_server/nbi_plugins/tfs_api/__init__.py @@ -22,7 +22,7 @@ from .Resources import ( PolicyRule, PolicyRuleIds, PolicyRules, Service, ServiceIds, Services, Slice, SliceIds, Slices, - Topologies, Topology, TopologyIds + Topologies, Topology, TopologyIds, TopologyDetails ) URL_PREFIX = '/tfs-api' @@ -30,38 +30,39 @@ URL_PREFIX = '/tfs-api' # Use 'path' type since some identifiers might contain char '/' and Flask is unable to recognize them in 'string' type. RESOURCES = [ # (endpoint_name, resource_class, resource_url) - ('api.context_ids', ContextIds, '/context_ids'), - ('api.contexts', Contexts, '/contexts'), - ('api.dummy_contexts', DummyContexts, '/dummy_contexts'), - ('api.context', Context, '/context/'), + ('api.context_ids', ContextIds, '/context_ids'), + ('api.contexts', Contexts, '/contexts'), + ('api.dummy_contexts', DummyContexts, '/dummy_contexts'), + ('api.context', Context, '/context/'), - ('api.topology_ids', TopologyIds, '/context//topology_ids'), - ('api.topologies', Topologies, '/context//topologies'), - ('api.topology', Topology, '/context//topology/'), + ('api.topology_ids', TopologyIds, '/context//topology_ids'), + ('api.topologies', Topologies, '/context//topologies'), + ('api.topology', Topology, '/context//topology/'), + ('api.topology_details', TopologyDetails, '/context//topology_details/'), - ('api.service_ids', ServiceIds, '/context//service_ids'), - ('api.services', Services, '/context//services'), - ('api.service', Service, '/context//service/'), + ('api.service_ids', ServiceIds, '/context//service_ids'), + ('api.services', Services, '/context//services'), + ('api.service', Service, '/context//service/'), - ('api.slice_ids', SliceIds, '/context//slice_ids'), - ('api.slices', Slices, '/context//slices'), - ('api.slice', Slice, '/context//slice/'), + ('api.slice_ids', SliceIds, '/context//slice_ids'), + ('api.slices', Slices, '/context//slices'), + ('api.slice', Slice, '/context//slice/'), - ('api.device_ids', DeviceIds, '/device_ids'), - ('api.devices', Devices, '/devices'), - ('api.device', Device, '/device/'), + ('api.device_ids', DeviceIds, '/device_ids'), + ('api.devices', Devices, '/devices'), + ('api.device', Device, '/device/'), - ('api.link_ids', LinkIds, '/link_ids'), - ('api.links', Links, '/links'), - ('api.link', Link, '/link/'), + ('api.link_ids', LinkIds, '/link_ids'), + ('api.links', Links, '/links'), + ('api.link', Link, '/link/'), - ('api.connection_ids', ConnectionIds, '/context//service//connection_ids'), - ('api.connections', Connections, '/context//service//connections'), - ('api.connection', Connection, '/connection/'), + ('api.connection_ids', ConnectionIds, '/context//service//connection_ids'), + ('api.connections', Connections, '/context//service//connections'), + ('api.connection', Connection, '/connection/'), - ('api.policyrule_ids', PolicyRuleIds, '/policyrule_ids'), - ('api.policyrules', PolicyRules, '/policyrules'), - ('api.policyrule', PolicyRule, '/policyrule/'), + ('api.policyrule_ids', PolicyRuleIds, '/policyrule_ids'), + ('api.policyrules', PolicyRules, '/policyrules'), + ('api.policyrule', PolicyRule, '/policyrule/'), ] def register_tfs_api(rest_server : RestServer): diff --git a/src/tests/ecoc24/.gitlab-ci.yml b/src/tests/ecoc24/.gitlab-ci.yml index af0646a0f..412b25450 100644 --- a/src/tests/ecoc24/.gitlab-ci.yml +++ b/src/tests/ecoc24/.gitlab-ci.yml @@ -56,10 +56,6 @@ end2end_test ecoc24: - microk8s status --wait-ready - kubectl get pods --all-namespaces - # Deploy Optical Device Node Agents - #- > - # docker network create -d bridge --subnet=172.254.253.0/24 --gateway=172.254.253.254 - # --ip-range=172.254.253.0/24 na-br # Configure TeraFlowSDN deployment @@ -70,36 +66,45 @@ end2end_test ecoc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - - source src/tests/${TEST_NAME}/deploy_specs_e2e.sh - # Deploy TeraFlowSDN + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/e2eorchestratorservice.yaml + #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/vntmservice.yaml + + + + # Deploy Optical TeraFlowSDN + - source src/tests/${TEST_NAME}/deploy_specs_opt.sh - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/qdb.sh - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - - ./src/tests/${TEST_NAME}/subscription_ws_e2e.sh + - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_opt.sh - - source src/tests/${TEST_NAME}/deploy_specs_opt.sh - # Deploy TeraFlowSDN + + # Deploy IP TeraFlowSDN + - source src/tests/${TEST_NAME}/deploy_specs_ip.sh - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/qdb.sh - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh + - ./src/tests/${TEST_NAME}/subscription_ws_ip.sh + - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_ip.sh - - source src/tests/${TEST_NAME}/deploy_specs_ip.sh - # Deploy TeraFlowSDN + # Deploy E2E TeraFlowSDN + - source src/tests/${TEST_NAME}/deploy_specs_e2e.sh - ./deploy/crdb.sh - ./deploy/nats.sh - ./deploy/qdb.sh - ./deploy/expose_dashboard.sh - ./deploy/tfs.sh - ./deploy/show.sh - - ./src/tests/${TEST_NAME}/subscription_ws_ip.sh - + - ./src/tests/${TEST_NAME}/subscription_ws_e2e.sh + - cp /var/teraflow/tfs_runtime_env_vars.sh /var/teraflow/tfs_runtime_env_vars_e2e.sh + # Run end-to-end tests diff --git a/src/tests/ecoc24/deploy_e2e.sh b/src/tests/ecoc24/deploy_e2e.sh index da7ef252c..a135835c7 100755 --- a/src/tests/ecoc24/deploy_e2e.sh +++ b/src/tests/ecoc24/deploy_e2e.sh @@ -25,7 +25,12 @@ kubectl apply -f src/tests/ecoc24/nginx-ingress-controller-e2e.yaml # Deploy TFS for E2E source src/tests/ecoc24/deploy_specs_e2e.sh + +# Change the name for the database +cp manifests/contextservice.yaml manifests/contextservice.yaml.bak +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfse2e_context"/}' manifests/contextservice.yaml ./deploy/all.sh +mv manifests/contextservice.yaml.bak manifests/contextservice.yaml #Configure Subscription WS ./src/tests/ecoc24/subscription_ws_e2e.sh diff --git a/src/tests/ecoc24/deploy_ip.sh b/src/tests/ecoc24/deploy_ip.sh index a6c5e8255..418a270bc 100755 --- a/src/tests/ecoc24/deploy_ip.sh +++ b/src/tests/ecoc24/deploy_ip.sh @@ -25,7 +25,12 @@ kubectl apply -f src/tests/ecoc24/nginx-ingress-controller-ip.yaml # Deploy TFS for IP source src/tests/ecoc24/deploy_specs_ip.sh + +# Change the name for the database +cp manifests/contextservice.yaml manifests/contextservice.yaml.bak +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfsip_context"/}' manifests/contextservice.yaml ./deploy/all.sh +mv manifests/contextservice.yaml.bak manifests/contextservice.yaml #Configure Subscription WS ./src/tests/ecoc24/subscription_ws_ip.sh diff --git a/src/tests/ecoc24/deploy_opt.sh b/src/tests/ecoc24/deploy_opt.sh index 3a9523768..ab2971fd5 100755 --- a/src/tests/ecoc24/deploy_opt.sh +++ b/src/tests/ecoc24/deploy_opt.sh @@ -25,5 +25,11 @@ kubectl apply -f src/tests/ecoc24/nginx-ingress-controller-opt.yaml # Deploy TFS for OPT source src/tests/ecoc24/deploy_specs_opt.sh + +# Change the name for the database +cp manifests/contextservice.yaml manifests/contextservice.yaml.bak +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfsopt_context"/}' manifests/contextservice.yaml ./deploy/all.sh +mv manifests/contextservice.yaml.bak manifests/contextservice.yaml + mv tfs_runtime_env_vars.sh tfs_runtime_env_vars_opt.sh diff --git a/src/tests/ecoc24/deploy_specs_e2e.sh b/src/tests/ecoc24/deploy_specs_e2e.sh index a8353db54..3b287ba89 100755 --- a/src/tests/ecoc24/deploy_specs_e2e.sh +++ b/src/tests/ecoc24/deploy_specs_e2e.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service nbi" +export TFS_COMPONENTS="context device pathcomp service nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -137,7 +137,7 @@ export CRDB_DATABASE="tfs_e2e" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="YES" +export CRDB_DROP_DATABASE_IF_EXISTS="NO" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" @@ -214,3 +214,4 @@ export KFK_SERVER_PORT="9092" # Set the flag to YES for redeploying of Apache Kafka export KFK_REDEPLOY="" + \ No newline at end of file diff --git a/src/tests/ecoc24/deploy_specs_ip.sh b/src/tests/ecoc24/deploy_specs_ip.sh index 52bd545ce..114289f4d 100755 --- a/src/tests/ecoc24/deploy_specs_ip.sh +++ b/src/tests/ecoc24/deploy_specs_ip.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service nbi" +export TFS_COMPONENTS="context device pathcomp service nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -137,7 +137,7 @@ export CRDB_DATABASE="tfs_ip" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="YES" +export CRDB_DROP_DATABASE_IF_EXISTS="NO" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" diff --git a/src/tests/ecoc24/deploy_specs_opt.sh b/src/tests/ecoc24/deploy_specs_opt.sh index 999fba045..4c7997f52 100755 --- a/src/tests/ecoc24/deploy_specs_opt.sh +++ b/src/tests/ecoc24/deploy_specs_opt.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service nbi" +export TFS_COMPONENTS="context device pathcomp service nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" @@ -137,7 +137,7 @@ export CRDB_DATABASE="tfs_ip" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="YES" +export CRDB_DROP_DATABASE_IF_EXISTS="NO" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" diff --git a/src/tests/ecoc24/descriptors/link_mapping.json b/src/tests/ecoc24/descriptors/descriptor_e2e.json similarity index 99% rename from src/tests/ecoc24/descriptors/link_mapping.json rename to src/tests/ecoc24/descriptors/descriptor_e2e.json index d2e9f11d7..41649d92e 100644 --- a/src/tests/ecoc24/descriptors/link_mapping.json +++ b/src/tests/ecoc24/descriptors/descriptor_e2e.json @@ -82,4 +82,5 @@ ] } ] - } \ No newline at end of file + } + \ No newline at end of file diff --git a/src/tests/ecoc24/descriptors/descriptor_ip.json b/src/tests/ecoc24/descriptors/descriptor_ip.json index b0909321c..11ad2d5c1 100644 --- a/src/tests/ecoc24/descriptors/descriptor_ip.json +++ b/src/tests/ecoc24/descriptors/descriptor_ip.json @@ -209,4 +209,5 @@ } ], "dummy_mode":true - } \ No newline at end of file + } + \ No newline at end of file diff --git a/src/tests/ecoc24/descriptors/descriptor_opt.json b/src/tests/ecoc24/descriptors/descriptor_opt.json index 320da8692..cfb86b966 100644 --- a/src/tests/ecoc24/descriptors/descriptor_opt.json +++ b/src/tests/ecoc24/descriptors/descriptor_opt.json @@ -799,4 +799,5 @@ ] } ] - } \ No newline at end of file + } + \ No newline at end of file diff --git a/src/tests/ecoc24/tests/test_functional_bootstrap_e2e.py b/src/tests/ecoc24/tests/test_functional_bootstrap_e2e.py new file mode 100644 index 000000000..cd72f5e94 --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_bootstrap_e2e.py @@ -0,0 +1,67 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors', 'descriptor_e2e.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_bootstrap( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + """ DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 """ + assert 1 == 1 diff --git a/src/tests/ecoc24/tests/test_functional_bootstrap.py b/src/tests/ecoc24/tests/test_functional_bootstrap_ip.py similarity index 100% rename from src/tests/ecoc24/tests/test_functional_bootstrap.py rename to src/tests/ecoc24/tests/test_functional_bootstrap_ip.py diff --git a/src/tests/ecoc24/tests/test_functional_bootstrap_opt.py b/src/tests/ecoc24/tests/test_functional_bootstrap_opt.py new file mode 100644 index 000000000..2f400c346 --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_bootstrap_opt.py @@ -0,0 +1,67 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os, time +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from tests.Fixtures import context_client, device_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors', 'descriptor_opt.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_scenario_bootstrap( + context_client : ContextClient, # pylint: disable=redefined-outer-name + device_client : DeviceClient, # pylint: disable=redefined-outer-name +) -> None: + validate_empty_scenario(context_client) + + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + results = descriptor_loader.process() + check_descriptor_load_results(results, descriptor_loader) + descriptor_loader.validate() + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + +def test_scenario_devices_enabled( + context_client : ContextClient, # pylint: disable=redefined-outer-name +) -> None: + """ + This test validates that the devices are enabled. + """ + """ DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED + + num_devices = -1 + num_devices_enabled, num_retry = 0, 0 + while (num_devices != num_devices_enabled) and (num_retry < 10): + time.sleep(1.0) + response = context_client.ListDevices(Empty()) + num_devices = len(response.devices) + num_devices_enabled = 0 + for device in response.devices: + if device.device_operational_status != DEVICE_OP_STATUS_ENABLED: continue + num_devices_enabled += 1 + LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices)) + num_retry += 1 """ + assert 1 == 1 diff --git a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py index d684e044e..b33705ed5 100644 --- a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py +++ b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py @@ -45,7 +45,7 @@ GET_EVENT_TIMEOUT = 0.5 class VNTMEventDispatcher(threading.Thread): def __init__(self, host, port) -> None: - LOGGER.debug('Creating VTNM connector...') + LOGGER.debug('Creating VNTM connector...') self.host = host self.port = port super().__init__(name='VNTMEventDispatcher', daemon=True) @@ -107,8 +107,8 @@ class VNTMEventDispatcher(threading.Thread): while not self._terminate.is_set(): event = events_collector.get_event(block=True, timeout=GET_EVENT_TIMEOUT) - LOGGER.info('Event type: {}'.format(event)) if event is None: continue + LOGGER.info('Event type: {}'.format(event)) LOGGER.debug('Received event: {}'.format(event)) topology_details = context_client.GetTopologyDetails(TopologyId(**topology_id)) -- GitLab From 63571f203fa590f481b4b8456473b686096d6509 Mon Sep 17 00:00:00 2001 From: mansoca Date: Tue, 12 Nov 2024 14:49:52 +0000 Subject: [PATCH 4/6] working version --- src/context/service/database/Link.py | 7 ++-- .../service/database/models/LinkModel.py | 8 ++++- .../service/database/models/enums/LinkType.py | 32 +++++++++++++++++++ .../E2EOrchestratorServiceServicerImpl.py | 22 +++++++------ src/e2e_orchestrator/service/__main__.py | 11 +++---- .../nbi_plugins/tfs_api/Resources.py | 16 ++++++++-- .../service/ServiceServiceServicerImpl.py | 8 ++--- src/tests/ecoc24/Dockerfile | 11 ++++--- src/tests/ecoc24/deploy_e2e.sh | 2 +- src/tests/ecoc24/deploy_ip.sh | 2 +- src/tests/ecoc24/deploy_opt.sh | 2 +- src/tests/ecoc24/deploy_specs_opt.sh | 4 +-- .../service/VNTManagerServiceServicerImpl.py | 20 ++++++------ 13 files changed, 99 insertions(+), 46 deletions(-) create mode 100644 src/context/service/database/models/enums/LinkType.py diff --git a/src/context/service/database/Link.py b/src/context/service/database/Link.py index deef3769c..673947f03 100644 --- a/src/context/service/database/Link.py +++ b/src/context/service/database/Link.py @@ -28,6 +28,7 @@ from .models.TopologyModel import TopologyLinkModel, TopologyModel from .uuids.EndPoint import endpoint_get_uuid from .uuids.Link import link_get_uuid from .Events import notify_event_context, notify_event_link, notify_event_topology +from .models.enums.LinkType import grpc_to_enum__link_type_enum LOGGER = logging.getLogger(__name__) @@ -67,8 +68,8 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) raw_link_name = request.name link_name = raw_link_uuid if len(raw_link_name) == 0 else raw_link_name link_uuid = link_get_uuid(request.link_id, link_name=link_name, allow_random=True) - - now = datetime.datetime.utcnow() + link_type = grpc_to_enum__link_type_enum(request.link_type) + now = datetime.datetime.now(datetime.timezone.utc) topology_uuids : Set[str] = set() related_topologies : List[Dict] = list() @@ -117,6 +118,7 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) link_data = [{ 'link_uuid' : link_uuid, 'link_name' : link_name, + 'link_type' : link_type, 'total_capacity_gbps' : total_capacity_gbps, 'used_capacity_gbps' : used_capacity_gbps, 'created_at' : now, @@ -129,6 +131,7 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) index_elements=[LinkModel.link_uuid], set_=dict( link_name = stmt.excluded.link_name, + link_type = stmt.excluded.link_type, total_capacity_gbps = stmt.excluded.total_capacity_gbps, used_capacity_gbps = stmt.excluded.used_capacity_gbps, updated_at = stmt.excluded.updated_at, diff --git a/src/context/service/database/models/LinkModel.py b/src/context/service/database/models/LinkModel.py index 423e39832..2de279a6e 100644 --- a/src/context/service/database/models/LinkModel.py +++ b/src/context/service/database/models/LinkModel.py @@ -13,17 +13,20 @@ # limitations under the License. import operator -from sqlalchemy import CheckConstraint, Column, DateTime, Float, ForeignKey, Integer, String +from sqlalchemy import CheckConstraint, Column, DateTime, Enum, Float, ForeignKey, Integer, String from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from typing import Dict from ._Base import _Base +from common.proto.context_pb2 import LinkTypeEnum +from .enums.LinkType import ORM_LinkTypeEnum class LinkModel(_Base): __tablename__ = 'link' link_uuid = Column(UUID(as_uuid=False), primary_key=True) link_name = Column(String, nullable=False) + link_type = Column(Enum(ORM_LinkTypeEnum), nullable=False) total_capacity_gbps = Column(Float, nullable=True) used_capacity_gbps = Column(Float, nullable=True) created_at = Column(DateTime, nullable=False) @@ -44,11 +47,14 @@ class LinkModel(_Base): result = { 'link_id' : self.dump_id(), 'name' : self.link_name, + 'link_type' : self.link_type.value, 'link_endpoint_ids': [ link_endpoint.endpoint.dump_id() for link_endpoint in sorted(self.link_endpoints, key=operator.attrgetter('position')) ], } + if self.link_type is None: + self.link_type = LinkTypeEnum.LINKTYPE_UNKNOWN if self.total_capacity_gbps is not None: attributes : Dict = result.setdefault('attributes', dict()) attributes.setdefault('total_capacity_gbps', self.total_capacity_gbps) diff --git a/src/context/service/database/models/enums/LinkType.py b/src/context/service/database/models/enums/LinkType.py new file mode 100644 index 000000000..1ac1a547f --- /dev/null +++ b/src/context/service/database/models/enums/LinkType.py @@ -0,0 +1,32 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum, functools +from common.proto.context_pb2 import LinkTypeEnum +from ._GrpcToEnum import grpc_to_enum + +# IMPORTANT: Entries of enum class ORM_DeviceDriverEnum should be named as in +# the proto files removing the prefixes. For example, proto item +# DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG should be included as +# OPENCONFIG. If item name does not match, automatic mapping of +# proto enums to database enums will fail. +class ORM_LinkTypeEnum(enum.Enum): + UNKNOWN = LinkTypeEnum.LINKTYPE_UNKNOWN + COPPER = LinkTypeEnum.LINKTYPE_COPPER + VIRTUAL_COPPER = LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER + OPTICAL = LinkTypeEnum.LINKTYPE_OPTICAL + VIRTUAL_OPTICAL = LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL + +grpc_to_enum__link_type_enum = functools.partial( + grpc_to_enum, LinkTypeEnum, ORM_LinkTypeEnum) diff --git a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py index ae9bfc7d2..c9681f708 100644 --- a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py +++ b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py @@ -80,7 +80,6 @@ class SubscriptionServer(Thread): LOGGER.debug("Received message from WebSocket: {}".format(message)) except Exception as ex: LOGGER.error('Exception receiving from WebSocket: {}'.format(ex)) - self._events_server() @@ -99,12 +98,13 @@ class SubscriptionServer(Thread): def _event_received(self, connection): + LOGGER.debug('Event received') for message in connection: message_json = json.loads(message) - # LOGGER.info("message_json: {}".format(message_json)) # Link creation if 'link_id' in message_json: + LOGGER.debug('Link creation') link = Link(**message_json) service = Service() @@ -114,12 +114,12 @@ class SubscriptionServer(Thread): service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED service_client.CreateService(service) - links = context_client.ListLinks(Empty()).links a_device_uuid = device_get_uuid(link.link_endpoint_ids[0].device_id) a_endpoint_uuid = endpoint_get_uuid(link.link_endpoint_ids[0])[2] z_device_uuid = device_get_uuid(link.link_endpoint_ids[1].device_id) z_endpoint_uuid = endpoint_get_uuid(link.link_endpoint_ids[1])[2] + links = context_client.ListLinks(Empty()).links for _link in links: for _endpoint_id in _link.link_endpoint_ids: if _endpoint_id.device_id.device_uuid.uuid == a_device_uuid and \ @@ -130,7 +130,9 @@ class SubscriptionServer(Thread): z_ep_id = _endpoint_id if (not 'a_ep_id' in locals()) or (not 'z_ep_id' in locals()): - error_msg = 'Could not get VNT link endpoints' + error_msg = f'Could not get VNT link endpoints\ + \n\ta_endpoint_uuid= {a_endpoint_uuid}\ + \n\tz_endpoint_uuid= {z_device_uuid}' LOGGER.error(error_msg) connection.send(error_msg) return @@ -138,20 +140,22 @@ class SubscriptionServer(Thread): service.service_endpoint_ids.append(copy.deepcopy(a_ep_id)) service.service_endpoint_ids.append(copy.deepcopy(z_ep_id)) - # service_client.UpdateService(service) + service_client.UpdateService(service) + re_svc = context_client.GetService(service.service_id) connection.send(grpc_message_to_json_string(link)) - # Link removal + context_client.SetLink(link) elif 'link_uuid' in message_json: + LOGGER.debug('Link removal') link_id = LinkId(**message_json) service_id = ServiceId() service_id.service_uuid.uuid = link_id.link_uuid.uuid service_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME - # service_client.DeleteService(service_id) + service_client.DeleteService(service_id) connection.send(grpc_message_to_json_string(link_id)) context_client.RemoveLink(link_id) - # Topology received else: + LOGGER.debug('Topology received') topology_details = TopologyDetails(**message_json) context = Context() @@ -187,12 +191,12 @@ class E2EOrchestratorServiceServicerImpl(E2EOrchestratorServiceServicer): i = 1 while True: try: - LOGGER.info(f'Retrieving external controller #{i}') ADD = str(get_setting(f'EXT_CONTROLLER{i}_ADD')) PORT = str(get_setting(f'EXT_CONTROLLER{i}_PORT')) except Exception as e: break try: + LOGGER.info(f'Retrieving external controller #{i}') url = f'http://{ADD}:{PORT}/tfs-api/context/{DEFAULT_CONTEXT_NAME}/topology_details/{DEFAULT_TOPOLOGY_NAME}' topo = requests.get(url).json() except Exception as e: diff --git a/src/e2e_orchestrator/service/__main__.py b/src/e2e_orchestrator/service/__main__.py index 0854aed2d..4c0a6d471 100644 --- a/src/e2e_orchestrator/service/__main__.py +++ b/src/e2e_orchestrator/service/__main__.py @@ -28,7 +28,10 @@ from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, from .E2EOrchestratorService import E2EOrchestratorService terminate = threading.Event() -LOGGER = None + +LOG_LEVEL = get_log_level() +logging.basicConfig(level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") +LOGGER = logging.getLogger(__name__) def signal_handler(signal, frame): # pylint: disable=redefined-outer-name @@ -37,12 +40,6 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name def main(): - global LOGGER # pylint: disable=global-statement - - log_level = get_log_level() - logging.basicConfig(level=log_level) - LOGGER = logging.getLogger(__name__) - signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) diff --git a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py index 484ccff66..0b99fba50 100644 --- a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py @@ -300,21 +300,31 @@ class Links(_Resource): ] class Link(_Resource): + @staticmethod + def _virtual_link(link): + virtual_types = {LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER, LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL} + if link.link_type in virtual_types: + return True + return False + + def get(self, link_uuid : str): return format_grpc_to_json(self.context_client.GetLink(grpc_link_id(link_uuid))) def put(self, link_uuid : str): link_json = request.get_json() link = grpc_link(link_json) - virtual_types = {LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER, LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL} if link_uuid != link.link_id.link_uuid.uuid: raise BadRequest('Mismatching link_uuid') - elif link.link_type in virtual_types: + elif self._virtual_link(link): link = grpc_link(link_json) return format_grpc_to_json(self.vntmanager_client.SetVirtualLink(link)) - return format_grpc_to_json(self.context_client.SetLink(grpc_link(link))) + return format_grpc_to_json(self.context_client.SetLink(link)) def delete(self, link_uuid : str): + link = self.context_client.GetLink(grpc_link_id(link_uuid)) + if self._virtual_link(link): + format_grpc_to_json(self.vntmanager_client.RemoveVirtualLink(grpc_link_id(link_uuid))) return format_grpc_to_json(self.context_client.RemoveLink(grpc_link_id(link_uuid))) class ConnectionIds(_Resource): diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 9120d475b..b6f8a7faf 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -384,10 +384,10 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): if "bandwidth" in constraint.custom.constraint_type: bitrate = int(float(constraint.custom.constraint_value)) break - - bitrate = int(float( - service.service_constraints[0].custom.constraint_value - )) + if service.service_constraints: + bitrate = int(float( + service.service_constraints[0].custom.constraint_value + )) if len(service.service_config.config_rules) > 0: c_rules_dict = json.loads( service.service_config.config_rules[0].custom.resource_value) diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile index 727abbd3a..f9a616e76 100644 --- a/src/tests/ecoc24/Dockerfile +++ b/src/tests/ecoc24/Dockerfile @@ -82,19 +82,22 @@ COPY src/tests/*.py ./tests/ COPY src/tests/ecoc24/__init__.py ./tests/ecoc24/__init__.py COPY src/tests/ecoc24/descriptors/descriptor_ip.json ./tests/ecoc24/descriptors/descriptor_ip.json COPY src/tests/ecoc24/descriptors/descriptor_opt.json ./tests/ecoc24/descriptors/descriptor_opt.json -COPY src/tests/ecoc24/descriptors/link_mapping.json ./tests/ecoc24/descriptors/link_mapping.json +COPY src/tests/ecoc24/descriptors/link_mapping.json ./tests/ecoc24/descriptors/descriptor_e2e.json COPY src/tests/ecoc24/tests/. ./tests/ecoc24/tests/ RUN tee ./run_tests.sh < None: - - time.sleep(5) events_collector = EventsCollector( context_client, log_events_received=True, - activate_context_collector = False, + activate_context_collector = True, activate_topology_collector = True, - activate_device_collector = False, - activate_link_collector = False, + activate_device_collector = True, + activate_link_collector = True, activate_service_collector = False, activate_slice_collector = False, activate_connection_collector = False,) @@ -92,7 +90,7 @@ class VNTMEventDispatcher(threading.Thread): LOGGER.info("Connecting to events server...: {}".format(url)) self.websocket = connect(url) except Exception as ex: - LOGGER.error('Error connecting to {}'.format(url)) + LOGGER.error(f'Error connecting to {url}\n\t{ex}') else: LOGGER.info('Connected to {}'.format(url)) context_id = json_context_id(DEFAULT_CONTEXT_NAME) @@ -108,12 +106,9 @@ class VNTMEventDispatcher(threading.Thread): while not self._terminate.is_set(): event = events_collector.get_event(block=True, timeout=GET_EVENT_TIMEOUT) if event is None: continue - LOGGER.info('Event type: {}'.format(event)) - LOGGER.debug('Received event: {}'.format(event)) + LOGGER.debug('Event type: {}'.format(event)) topology_details = context_client.GetTopologyDetails(TopologyId(**topology_id)) - to_send = grpc_message_to_json_string(topology_details) - self.send_msg(to_send) LOGGER.info('Exiting') @@ -135,6 +130,8 @@ class VNTManagerServiceServicerImpl(VNTManagerServiceServicer): self.event_dispatcher = VNTMEventDispatcher(request.host, int(request.port)) self.host = request.host self.port = request.port + LOGGER.info('sleeping 5...') + time.sleep(5) self.event_dispatcher.start() return reply @@ -158,12 +155,13 @@ class VNTManagerServiceServicerImpl(VNTManagerServiceServicer): link = Link(**message_json) context_client.SetLink(link) except Exception as e: - LOGGER.error('Exception setting virtual link={}\n\t{}'.format(request.link_id.link_uuid.uuid, e)) + LOGGER.error(f'Exception setting virtual link={request.link_id.link_uuid.uuid}\n\t{e}') return request.link_id @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def RemoveVirtualLink(self, request : LinkId, context : grpc.ServicerContext) -> Empty: try: + LOGGER.debug('Removing virtual link') self.event_dispatcher.send_msg(grpc_message_to_json_string(request)) # deconfigure('CSGW1', 'xe5', 'CSGW2', 'xe5', 'ecoc2024-1') response = self.event_dispatcher.recv_msg() -- GitLab From 7235b7e7e6ab592b012b0365de6260c7271556df Mon Sep 17 00:00:00 2001 From: mansoca Date: Tue, 12 Nov 2024 15:13:33 +0000 Subject: [PATCH 5/6] check cicd --- src/tests/ecoc24/Dockerfile | 14 ++-- src/tests/ecoc24/tests/create_service.py | 27 +++++++ src/tests/ecoc24/tests/delete_service.py | 10 +++ .../ecoc24/tests/test_functional_cleanup.py | 44 ++++++++++ .../tests/test_functional_create_service.py | 74 +++++++++++++++++ .../tests/test_functional_delete_service.py | 80 +++++++++++++++++++ 6 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 src/tests/ecoc24/tests/create_service.py create mode 100644 src/tests/ecoc24/tests/delete_service.py create mode 100644 src/tests/ecoc24/tests/test_functional_cleanup.py create mode 100644 src/tests/ecoc24/tests/test_functional_create_service.py create mode 100644 src/tests/ecoc24/tests/test_functional_delete_service.py diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile index f9a616e76..52558e338 100644 --- a/src/tests/ecoc24/Dockerfile +++ b/src/tests/ecoc24/Dockerfile @@ -87,16 +87,14 @@ COPY src/tests/ecoc24/tests/. ./tests/ecoc24/tests/ RUN tee ./run_tests.sh < None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) diff --git a/src/tests/ecoc24/tests/test_functional_create_service.py b/src/tests/ecoc24/tests/test_functional_create_service.py new file mode 100644 index 000000000..d4f21978c --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_create_service.py @@ -0,0 +1,74 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceStatusEnum, ServiceTypeEnum +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from service.client.ServiceClient import ServiceClient +from tests.Fixtures import context_client, device_client, service_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors', 'virtual_link.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_creation_bidir( + context_client : ContextClient, # pylint: disable=redefined-outer-name + # device_client : DeviceClient, # pylint: disable=redefined-outer-name + # service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + # Load descriptors and validate the base scenario + # descriptor_loader = DescriptorLoader( + # descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client, + # service_client=service_client + # ) + # results = descriptor_loader.process() + # check_descriptor_load_results(results, descriptor_loader) + + import create_service + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 1 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY: + assert len(response.connections) == 2 + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) diff --git a/src/tests/ecoc24/tests/test_functional_delete_service.py b/src/tests/ecoc24/tests/test_functional_delete_service.py new file mode 100644 index 000000000..475c58e61 --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_delete_service.py @@ -0,0 +1,80 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set, Tuple +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceId, ServiceStatusEnum, ServiceTypeEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Service import json_service_id +from context.client.ContextClient import ContextClient +from service.client.ServiceClient import ServiceClient +from tests.Fixtures import context_client, service_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_removal_bidir( + context_client : ContextClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 1 + + context_service_uuids : Set[Tuple[str, str]] = set() + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY: + assert len(response.connections) == 2 + context_uuid = service_id.context_id.context_uuid.uuid + service_uuid = service_id.service_uuid.uuid + context_service_uuids.add((context_uuid, service_uuid)) + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) + + # Identify service to delete + assert len(context_service_uuids) == 1 + context_uuid, service_uuid = set(context_service_uuids).pop() + + # Delete Service + # service_client.DeleteService(ServiceId(**json_service_id(service_uuid, json_context_id(context_uuid)))) + + import delete_service + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 -- GitLab From 74fe9c3508426ea074e1f5d9abc65a350e0b32a8 Mon Sep 17 00:00:00 2001 From: mansoca Date: Tue, 12 Nov 2024 15:13:33 +0000 Subject: [PATCH 6/6] check cicd --- src/tests/ecoc24/Dockerfile | 16 ++-- src/tests/ecoc24/tests/create_service.py | 27 +++++++ src/tests/ecoc24/tests/delete_service.py | 10 +++ .../ecoc24/tests/test_functional_cleanup.py | 44 ++++++++++ .../tests/test_functional_create_service.py | 74 +++++++++++++++++ .../tests/test_functional_delete_service.py | 80 +++++++++++++++++++ 6 files changed, 242 insertions(+), 9 deletions(-) create mode 100644 src/tests/ecoc24/tests/create_service.py create mode 100644 src/tests/ecoc24/tests/delete_service.py create mode 100644 src/tests/ecoc24/tests/test_functional_cleanup.py create mode 100644 src/tests/ecoc24/tests/test_functional_create_service.py create mode 100644 src/tests/ecoc24/tests/test_functional_delete_service.py diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile index f9a616e76..822e2a0c0 100644 --- a/src/tests/ecoc24/Dockerfile +++ b/src/tests/ecoc24/Dockerfile @@ -82,21 +82,19 @@ COPY src/tests/*.py ./tests/ COPY src/tests/ecoc24/__init__.py ./tests/ecoc24/__init__.py COPY src/tests/ecoc24/descriptors/descriptor_ip.json ./tests/ecoc24/descriptors/descriptor_ip.json COPY src/tests/ecoc24/descriptors/descriptor_opt.json ./tests/ecoc24/descriptors/descriptor_opt.json -COPY src/tests/ecoc24/descriptors/link_mapping.json ./tests/ecoc24/descriptors/descriptor_e2e.json +COPY src/tests/ecoc24/descriptors/descriptor_e2e.json ./tests/ecoc24/descriptors/descriptor_e2e.json COPY src/tests/ecoc24/tests/. ./tests/ecoc24/tests/ RUN tee ./run_tests.sh < None: + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 + + # Load descriptors and validate the base scenario + descriptor_loader = DescriptorLoader( + descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client) + descriptor_loader.validate() + descriptor_loader.unload() + validate_empty_scenario(context_client) diff --git a/src/tests/ecoc24/tests/test_functional_create_service.py b/src/tests/ecoc24/tests/test_functional_create_service.py new file mode 100644 index 000000000..d4f21978c --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_create_service.py @@ -0,0 +1,74 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, os +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceStatusEnum, ServiceTypeEnum +from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from service.client.ServiceClient import ServiceClient +from tests.Fixtures import context_client, device_client, service_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors', 'virtual_link.json') +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_creation_bidir( + context_client : ContextClient, # pylint: disable=redefined-outer-name + # device_client : DeviceClient, # pylint: disable=redefined-outer-name + # service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + # Load descriptors and validate the base scenario + # descriptor_loader = DescriptorLoader( + # descriptors_file=DESCRIPTOR_FILE, context_client=context_client, device_client=device_client, + # service_client=service_client + # ) + # results = descriptor_loader.process() + # check_descriptor_load_results(results, descriptor_loader) + + import create_service + + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 1 + + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY: + assert len(response.connections) == 2 + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) diff --git a/src/tests/ecoc24/tests/test_functional_delete_service.py b/src/tests/ecoc24/tests/test_functional_delete_service.py new file mode 100644 index 000000000..475c58e61 --- /dev/null +++ b/src/tests/ecoc24/tests/test_functional_delete_service.py @@ -0,0 +1,80 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Set, Tuple +from common.Constants import DEFAULT_CONTEXT_NAME +from common.proto.context_pb2 import ContextId, ServiceId, ServiceStatusEnum, ServiceTypeEnum +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Service import json_service_id +from context.client.ContextClient import ContextClient +from service.client.ServiceClient import ServiceClient +from tests.Fixtures import context_client, service_client # pylint: disable=unused-import + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME)) + +def test_service_removal_bidir( + context_client : ContextClient, # pylint: disable=redefined-outer-name + service_client : ServiceClient, # pylint: disable=redefined-outer-name +): + # Verify the scenario has 1 service and 0 slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 1 + assert len(response.slice_ids) == 0 + + # Check there are no slices + response = context_client.ListSlices(ADMIN_CONTEXT_ID) + LOGGER.warning('Slices[{:d}] = {:s}'.format(len(response.slices), grpc_message_to_json_string(response))) + assert len(response.slices) == 0 + + # Check there is 1 service + response = context_client.ListServices(ADMIN_CONTEXT_ID) + LOGGER.warning('Services[{:d}] = {:s}'.format(len(response.services), grpc_message_to_json_string(response))) + assert len(response.services) == 1 + + context_service_uuids : Set[Tuple[str, str]] = set() + for service in response.services: + service_id = service.service_id + assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE + + response = context_client.ListConnections(service_id) + LOGGER.warning(' ServiceId[{:s}] => Connections[{:d}] = {:s}'.format( + grpc_message_to_json_string(service_id), len(response.connections), grpc_message_to_json_string(response))) + + if service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY: + assert len(response.connections) == 2 + context_uuid = service_id.context_id.context_uuid.uuid + service_uuid = service_id.service_uuid.uuid + context_service_uuids.add((context_uuid, service_uuid)) + else: + str_service = grpc_message_to_json_string(service) + raise Exception('Unexpected ServiceType: {:s}'.format(str_service)) + + # Identify service to delete + assert len(context_service_uuids) == 1 + context_uuid, service_uuid = set(context_service_uuids).pop() + + # Delete Service + # service_client.DeleteService(ServiceId(**json_service_id(service_uuid, json_context_id(context_uuid)))) + + import delete_service + + # Verify the scenario has no services/slices + response = context_client.GetContext(ADMIN_CONTEXT_ID) + assert len(response.service_ids) == 0 + assert len(response.slice_ids) == 0 -- GitLab