diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c3626b2f576dae2135de2030c01ada133aa89d7..095e542dfa8c5eca947a2931e6a9348aef70d838 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,12 +35,13 @@ include: - local: '/src/ztp/.gitlab-ci.yml' - local: '/src/policy/.gitlab-ci.yml' - local: '/src/forecaster/.gitlab-ci.yml' - #- local: '/src/webui/.gitlab-ci.yml' + - local: '/src/webui/.gitlab-ci.yml' #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml' #- local: '/src/l3_attackmitigator/.gitlab-ci.yml' - #- local: '/src/slice/.gitlab-ci.yml' + - local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' - local: '/src/load_generator/.gitlab-ci.yml' + - local: '/src/bgpls_speaker/.gitlab-ci.yml' diff --git a/manifests/bgpls_speakerservice.yaml b/manifests/bgpls_speakerservice.yaml index 0eca04d9a570ce44b5b3e9e05e8f0ac40cde8b7a..c0052660540d04cb363d409a819084f84deebc23 100644 --- a/manifests/bgpls_speakerservice.yaml +++ b/manifests/bgpls_speakerservice.yaml @@ -45,11 +45,11 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:20030"] resources: requests: - cpu: 50m - memory: 64Mi + cpu: 250m + memory: 128Mi limits: - cpu: 500m - memory: 512Mi + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service diff --git a/src/bgpls_speaker/.gitlab-ci.yml b/src/bgpls_speaker/.gitlab-ci.yml index 4c461ce56dfe6baf1716eb6deb070288c848faa2..68b04f8e7d12812c3d19a269a00d5337874d7762 100644 --- a/src/bgpls_speaker/.gitlab-ci.yml +++ b/src/bgpls_speaker/.gitlab-ci.yml @@ -13,7 +13,7 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -build service: +build bgpls_speaker: variables: IMAGE_NAME: 'bgpls_speaker' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) @@ -39,147 +39,29 @@ build service: - .gitlab-ci.yml # Apply unit test to the component -unit_test service: +unit_test bgpls_speaker: variables: IMAGE_NAME: 'bgpls_speaker' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) stage: unit_test needs: - - build service + - build bgpls_speaker before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi - - # Context-related - - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi - - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi - - # Device-related - - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi - - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi - - # Pathcomp-related - - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi - - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi - - # Service-related - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi - script: - - docker pull "cockroachdb/cockroach:latest-v22.2" - - docker pull "nats:2.9" - - docker pull "$CI_REGISTRY_IMAGE/context:$IMAGE_TAG" - - docker pull "$CI_REGISTRY_IMAGE/device:$IMAGE_TAG" - - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG" - - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - # Context preparation - - docker volume create crdb - - > - docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 - --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 - --volume "crdb:/cockroach/cockroach-data" - cockroachdb/cockroach:latest-v22.2 start-single-node - - > - docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222 - nats:2.9 --http_port 8222 --user tfs --pass tfs123 - - echo "Waiting for initialization..." - - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - - docker logs crdb - - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done - - docker logs nats - - docker ps -a - - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CRDB_ADDRESS - - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $NATS_ADDRESS - - > - docker run --name context -d -p 1010:1010 - --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --env "MB_BACKEND=nats" - --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/context:$IMAGE_TAG - - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $CONTEXTSERVICE_SERVICE_HOST - - # Device preparation - - > - docker run --name device -d -p 2020:2020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/device:$IMAGE_TAG - - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $DEVICESERVICE_SERVICE_HOST - - # PathComp preparation - - > - docker run --name pathcomp-backend -d -p 8081:8081 - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG - - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMP_BACKEND_HOST - - sleep 1 - - > - docker run --name pathcomp-frontend -d -p 10020:10020 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}" - --env "PATHCOMP_BACKEND_PORT=8081" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG - - sleep 1 - - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $PATHCOMPSERVICE_SERVICE_HOST - - # Service preparation - - > - docker run --name $IMAGE_NAME -d -p 3030:3030 - --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}" - --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}" - --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" - --network=teraflowbridge - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - - # Check status before the tests + - docker run --name $IMAGE_NAME -d -p 20030:20030 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - - docker logs context - - docker logs device - - docker logs pathcomp-frontend - - docker logs pathcomp-backend - docker logs $IMAGE_NAME - - # Run the tests - - > - docker exec -i $IMAGE_NAME bash -c - "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - # Check status after the tests - - docker ps -a - - docker logs context - - docker logs device - - docker logs pathcomp-frontend - - docker logs pathcomp-backend - - docker logs $IMAGE_NAME - - docker rm -f $IMAGE_NAME - - docker rm -f pathcomp-frontend - - docker rm -f pathcomp-backend - - docker rm -f device - - docker rm -f context - - - docker rm -f $IMAGE_NAME crdb nats - - docker volume rm -f crdb - docker network rm teraflowbridge - - docker volume prune --force - - docker image prune --force - rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' @@ -191,20 +73,19 @@ unit_test service: - src/$IMAGE_NAME/tests/*.py - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml - artifacts: when: always reports: junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml ## Deployment of the service in Kubernetes Cluster -#deploy service: +#deploy bgpls_speaker: # variables: -# IMAGE_NAME: 'service' # name of the microservice +# IMAGE_NAME: 'bgpls_speaker' # name of the microservice # IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) # stage: deploy # needs: -# - unit test service +# - unit test bgpls_speaker # # - integ_test execute # script: # - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' diff --git a/src/bgpls_speaker/Dockerfile b/src/bgpls_speaker/Dockerfile index dec0c6590efd0e3aa032bcda0bda2e533c0dd0fd..87463943af647188791e5d162b2af416df867285 100644 --- a/src/bgpls_speaker/Dockerfile +++ b/src/bgpls_speaker/Dockerfile @@ -101,10 +101,8 @@ COPY --from=build /netphony-topology/target/bgp-ls-speaker-jar-with-dependencies # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ -COPY src/device/. device/ -COPY src/pathcomp/frontend/. pathcomp/frontend/ -COPY src/service/. service/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ COPY src/bgpls_speaker/. bgpls_speaker/ # Start the service diff --git a/src/bgpls_speaker/requirements.in b/src/bgpls_speaker/requirements.in index 9dd4a0a687a74e627aef1a89ec491a0f04ab1c91..3af49b784a205ac6aada306219f5a2e658beea27 100644 --- a/src/bgpls_speaker/requirements.in +++ b/src/bgpls_speaker/requirements.in @@ -16,6 +16,7 @@ anytree==2.8.0 APScheduler==3.8.1 ncclient==0.6.13 python-json-logger==2.0.2 +lxml==4.9.1 pytz==2021.3 xmltodict==0.12.0 grpcio==1.47.* diff --git a/src/bgpls_speaker/service/BgplsServiceServicerImpl.py b/src/bgpls_speaker/service/BgplsServiceServicerImpl.py index ed4e382563c93a9d5c3910aba398168a1c3f06d9..79862ee3b1204af105c5bd75c233f80ff3a9cf00 100644 --- a/src/bgpls_speaker/service/BgplsServiceServicerImpl.py +++ b/src/bgpls_speaker/service/BgplsServiceServicerImpl.py @@ -13,20 +13,16 @@ # limitations under the License. import grpc, json, logging -from typing import Optional,List, Tuple, Union +from typing import List, Tuple, Union from bgpls_speaker.service.tools.DiscoveredDBManager import DiscoveredDBManager, GetContextDevices from bgpls_speaker.service.tools.GrpcServer import GrpcServer from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.method_wrappers.ServiceExceptions import AlreadyExistsException, InvalidArgumentException -from common.proto.context_pb2 import (DeviceId, Empty, EndPointId, Link, LinkId, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum, TopologyId,ContextId,Topology - ,Device,DeviceDriverEnum, Uuid) -from common.proto.pathcomp_pb2 import PathCompRequest -from common.proto.service_pb2_grpc import ServiceServiceServicer -from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string +from common.proto.context_pb2 import DeviceId, Empty, EndPointId, Link, LinkId, Uuid from context.client.ContextClient import ContextClient -from pathcomp.frontend.client.PathCompClient import PathCompClient -from common.proto.bgpls_pb2 import (BgplsSpeaker,BgplsSpeakerId, DiscoveredDeviceList,DiscoveredDevice, - DiscoveredLink,DiscoveredLinkList,NodeDescriptors,BgplsSpeakerList) +from common.proto.bgpls_pb2 import ( + BgplsSpeaker, BgplsSpeakerId, DiscoveredDeviceList, DiscoveredDevice, + DiscoveredLink, DiscoveredLinkList, NodeDescriptors, BgplsSpeakerList +) from common.proto.bgpls_pb2_grpc import BgplsServiceServicer def json_to_list(json_str : str) -> List[Union[str, Tuple[str, str]]]: diff --git a/src/bgpls_speaker/service/__main__.py b/src/bgpls_speaker/service/__main__.py index 99d64cfb6b37f9d3b89c8ceba231dc03ab9e6451..a56449eafb0eb40a31768e16f0fdcb1fa1dc72f8 100644 --- a/src/bgpls_speaker/service/__main__.py +++ b/src/bgpls_speaker/service/__main__.py @@ -12,16 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, signal, sys, threading, time +import logging, signal, sys, threading from prometheus_client import start_http_server -from common.Constants import ServiceNameEnum -from common.Settings import ( - ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port, - wait_for_environment_variables) -from context.client.ContextClient import ContextClient +from common.Settings import get_log_level, get_metrics_port from .tools.DiscoveredDBManager import DiscoveredDBManager from .BgplsService import BgplsService -from .tools.JavaRunner import JavaRunner from .tools.GrpcServer import GrpcServer terminate = threading.Event() diff --git a/src/bgpls_speaker/tests/test_unitary.py b/src/bgpls_speaker/tests/test_unitary.py new file mode 100644 index 0000000000000000000000000000000000000000..b8aae5c680e2fe65796fb4417e72a0ae4f925510 --- /dev/null +++ b/src/bgpls_speaker/tests/test_unitary.py @@ -0,0 +1,18 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) diff --git a/src/device/Dockerfile b/src/device/Dockerfile index 909ae3bd31817401412629cbc04b5c0577b40355..b230350c7b331eefbb6d4ac0f9bf3b5b46364254 100644 --- a/src/device/Dockerfile +++ b/src/device/Dockerfile @@ -62,9 +62,11 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ COPY src/device/. device/ -COPY src/monitoring/. monitoring/ RUN mkdir -p tests/tools/mock_ietf_actn_sdn_ctrl RUN touch tests/__init__.py diff --git a/src/pathcomp/frontend/Dockerfile b/src/pathcomp/frontend/Dockerfile index 955844cf4d80b39fc0913c9c523fd1267ca0fb1d..b54a49cabab79112e65b28a7e5e28d5a17056d67 100644 --- a/src/pathcomp/frontend/Dockerfile +++ b/src/pathcomp/frontend/Dockerfile @@ -54,26 +54,28 @@ RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages -RUN mkdir -p /var/teraflow/pathcomp -WORKDIR /var/teraflow/pathcomp +RUN mkdir -p /var/teraflow/pathcomp/frontend +WORKDIR /var/teraflow/pathcomp/frontend COPY src/pathcomp/frontend/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow +COPY src/pathcomp/__init__.py pathcomp/__init__.py +COPY src/pathcomp/frontend/. pathcomp/frontend/ COPY src/context/__init__.py context/__init__.py COPY src/context/client/. context/client/ COPY src/device/__init__.py device/__init__.py COPY src/device/client/. device/client/ -COPY src/forecaster/. forecaster/ +COPY src/forecaster/__init__.py forecaster/__init__.py +COPY src/forecaster/client/. forecaster/client/ COPY src/monitoring/__init__.py monitoring/__init__.py COPY src/monitoring/client/. monitoring/client/ COPY src/service/__init__.py service/__init__.py COPY src/service/client/. service/client/ COPY src/slice/__init__.py slice/__init__.py COPY src/slice/client/. slice/client/ -COPY src/pathcomp/. pathcomp/ # Start the service ENTRYPOINT ["python", "-m", "pathcomp.frontend.service"] diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 85454350f1eb9413af0ef995386868d09222121c..aed4ce64c60a7cfca5b80dc46e03bad18a017ca9 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -62,7 +62,6 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/service/. service/ COPY src/context/__init__.py context/__init__.py COPY src/context/client/. context/client/ COPY src/device/__init__.py device/__init__.py @@ -71,6 +70,7 @@ COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/ COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/ +COPY src/service/. service/ # Start the service ENTRYPOINT ["python", "-m", "service.service"] diff --git a/src/slice/Dockerfile b/src/slice/Dockerfile index 94d5fc040dff083407cae514ed0363b5d2ea9bb1..da380eabc7f8a54b235b87b338ad82a4ead88b4d 100644 --- a/src/slice/Dockerfile +++ b/src/slice/Dockerfile @@ -62,10 +62,15 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ -COPY src/interdomain/. interdomain/ -COPY src/pathcomp/. pathcomp/ -COPY src/service/. service/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/interdomain/__init__.py interdomain/__init__.py +COPY src/interdomain/client/. interdomain/client/ +COPY src/pathcomp/__init__.py pathcomp/__init__.py +COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py +COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/ +COPY src/service/__init__.py service/__init__.py +COPY src/service/client/. service/client/ COPY src/slice/. slice/ # Start the service