Merge branch 'feat/tid-bgp-speaker' of...

Merge branch 'feat/tid-bgp-speaker' of https://labs.etsi.org/rep/tfs/controller into feat/tid-bgp-speaker

.gitlab-ci.yml

@@ -35,12 +35,13 @@ include:
   - local: '/src/ztp/.gitlab-ci.yml'
   - local: '/src/policy/.gitlab-ci.yml'
   - local: '/src/forecaster/.gitlab-ci.yml'
-  #- local: '/src/webui/.gitlab-ci.yml'
+  - local: '/src/webui/.gitlab-ci.yml'
   #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'
   #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml'
   #- local: '/src/l3_attackmitigator/.gitlab-ci.yml'
-  #- local: '/src/slice/.gitlab-ci.yml'
+  - local: '/src/slice/.gitlab-ci.yml'
   #- local: '/src/interdomain/.gitlab-ci.yml'
   - local: '/src/pathcomp/.gitlab-ci.yml'
   #- local: '/src/dlt/.gitlab-ci.yml'
   - local: '/src/load_generator/.gitlab-ci.yml'
+  - local: '/src/bgpls_speaker/.gitlab-ci.yml'

manifests/bgpls_speakerservice.yaml

@@ -45,11 +45,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:20030"]
         resources:
           requests:
-            cpu: 50m
-            memory: 64Mi
+            cpu: 250m
+            memory: 128Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 1000m
+            memory: 1024Mi
 ---
 apiVersion: v1
 kind: Service

src/bgpls_speaker/.gitlab-ci.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Build, tag, and push the Docker image to the GitLab Docker registry
-build service:
+build bgpls_speaker:
   variables:
     IMAGE_NAME: 'bgpls_speaker' # name of the microservice
     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
@@ -39,147 +39,29 @@ build service:
       - .gitlab-ci.yml
 
 # Apply unit test to the component
-unit_test service:
+unit_test bgpls_speaker:
   variables:
     IMAGE_NAME: 'bgpls_speaker' # name of the microservice
     IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
   stage: unit_test
   needs:
-    - build service
+    - build bgpls_speaker
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
     - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi
-
-    # Context-related
-    - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi
-    - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi
-    - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi
-
-    # Device-related
-    - if docker container ls | grep context; then docker rm -f context; else echo "context image is not in the system"; fi
-    - if docker container ls | grep device; then docker rm -f device; else echo "device image is not in the system"; fi
-
-    # Pathcomp-related
-    - if docker container ls | grep pathcomp-frontend; then docker rm -f pathcomp-frontend; else echo "pathcomp-frontend image is not in the system"; fi
-    - if docker container ls | grep pathcomp-backend; then docker rm -f pathcomp-backend; else echo "pathcomp-backend image is not in the system"; fi
-
-    # Service-related
     - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
-
   script:
-    - docker pull "cockroachdb/cockroach:latest-v22.2"
-    - docker pull "nats:2.9"
-    - docker pull "$CI_REGISTRY_IMAGE/context:$IMAGE_TAG"
-    - docker pull "$CI_REGISTRY_IMAGE/device:$IMAGE_TAG"
-    - docker pull "$CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG"
-    - docker pull "$CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG"
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
-
-    # Context preparation
-    - docker volume create crdb
-    - >
-      docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080
-      --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123
-      --volume "crdb:/cockroach/cockroach-data"
-      cockroachdb/cockroach:latest-v22.2 start-single-node
-    - >
-      docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222
-      nats:2.9 --http_port 8222 --user tfs --pass tfs123
-    - echo "Waiting for initialization..."
-    - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done
-    - docker logs crdb
-    - while ! docker logs nats 2>&1 | grep -q 'Server is ready'; do sleep 1; done
-    - docker logs nats
-    - docker ps -a
-    - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $CRDB_ADDRESS
-    - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $NATS_ADDRESS
-    - >
-      docker run --name context -d -p 1010:1010
-      --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require"
-      --env "MB_BACKEND=nats"
-      --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222"
-      --network=teraflowbridge
-      $CI_REGISTRY_IMAGE/context:$IMAGE_TAG
-    - CONTEXTSERVICE_SERVICE_HOST=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $CONTEXTSERVICE_SERVICE_HOST
-
-    # Device preparation
-    - >
-      docker run --name device -d -p 2020:2020
-      --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}"
-      --network=teraflowbridge
-      $CI_REGISTRY_IMAGE/device:$IMAGE_TAG
-    - DEVICESERVICE_SERVICE_HOST=$(docker inspect device --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $DEVICESERVICE_SERVICE_HOST
-
-    # PathComp preparation
-    - >
-      docker run --name pathcomp-backend -d -p 8081:8081
-      --network=teraflowbridge
-      $CI_REGISTRY_IMAGE/pathcomp-backend:$IMAGE_TAG
-    - PATHCOMP_BACKEND_HOST=$(docker inspect pathcomp-backend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $PATHCOMP_BACKEND_HOST
-    - sleep 1
-    - >
-      docker run --name pathcomp-frontend -d -p 10020:10020
-      --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}"
-      --env "PATHCOMP_BACKEND_HOST=${PATHCOMP_BACKEND_HOST}"
-      --env "PATHCOMP_BACKEND_PORT=8081"
-      --network=teraflowbridge
-      $CI_REGISTRY_IMAGE/pathcomp-frontend:$IMAGE_TAG
-    - sleep 1
-    - PATHCOMPSERVICE_SERVICE_HOST=$(docker inspect pathcomp-frontend --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
-    - echo $PATHCOMPSERVICE_SERVICE_HOST
-
-    # Service preparation
-    - >
-      docker run --name $IMAGE_NAME -d -p 3030:3030
-      --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXTSERVICE_SERVICE_HOST}"
-      --env "DEVICESERVICE_SERVICE_HOST=${DEVICESERVICE_SERVICE_HOST}"
-      --env "PATHCOMPSERVICE_SERVICE_HOST=${PATHCOMPSERVICE_SERVICE_HOST}"
-      --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results"
-      --network=teraflowbridge
-      $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
-
-    # Check status before the tests
+    - docker run --name $IMAGE_NAME -d -p 20030:20030  -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - sleep 5
     - docker ps -a
-    - docker logs context
-    - docker logs device
-    - docker logs pathcomp-frontend
-    - docker logs pathcomp-backend
     - docker logs $IMAGE_NAME
-
-    # Run the tests
-    - >
-      docker exec -i $IMAGE_NAME bash -c
-      "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml"
+    - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml"
     - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
-
   coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
   after_script:
-    # Check status after the tests
-    - docker ps -a
-    - docker logs context
-    - docker logs device
-    - docker logs pathcomp-frontend
-    - docker logs pathcomp-backend
-    - docker logs $IMAGE_NAME
-
     - docker rm -f $IMAGE_NAME
-    - docker rm -f pathcomp-frontend
-    - docker rm -f pathcomp-backend
-    - docker rm -f device
-    - docker rm -f context
-
-    - docker rm -f $IMAGE_NAME crdb nats
-    - docker volume rm -f crdb
     - docker network rm teraflowbridge
-    - docker volume prune --force
-    - docker image prune --force
-
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
@@ -191,20 +73,19 @@ unit_test service:
       - src/$IMAGE_NAME/tests/*.py
       - manifests/${IMAGE_NAME}service.yaml
       - .gitlab-ci.yml
-
   artifacts:
       when: always
       reports:
         junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
 
 ## Deployment of the service in Kubernetes Cluster
-#deploy service:
+#deploy bgpls_speaker:
 #  variables:
-#    IMAGE_NAME: 'service' # name of the microservice
+#    IMAGE_NAME: 'bgpls_speaker' # name of the microservice
 #    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
 #  stage: deploy
 #  needs:
-#    - unit test service
+#    - unit test bgpls_speaker
 #    # - integ_test execute
 #  script:
 #    - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml'

src/bgpls_speaker/Dockerfile

@@ -101,10 +101,8 @@ COPY --from=build /netphony-topology/target/bgp-ls-speaker-jar-with-dependencies
 
 # Add component files into working directory
 WORKDIR /var/teraflow
-COPY src/context/. context/
-COPY src/device/. device/
-COPY src/pathcomp/frontend/. pathcomp/frontend/
-COPY src/service/. service/
+COPY src/context/__init__.py context/__init__.py
+COPY src/context/client/. context/client/
 COPY src/bgpls_speaker/. bgpls_speaker/
 
 # Start the service

src/bgpls_speaker/requirements.in

@@ -16,6 +16,7 @@ anytree==2.8.0
 APScheduler==3.8.1
 ncclient==0.6.13
 python-json-logger==2.0.2
+lxml==4.9.1
 pytz==2021.3
 xmltodict==0.12.0
 grpcio==1.47.*

src/bgpls_speaker/service/BgplsServiceServicerImpl.py

@@ -13,20 +13,16 @@
 # limitations under the License.
 
 import grpc, json, logging
-from typing import Optional,List, Tuple, Union
+from typing import List, Tuple, Union
 from bgpls_speaker.service.tools.DiscoveredDBManager import  DiscoveredDBManager, GetContextDevices
 from bgpls_speaker.service.tools.GrpcServer import GrpcServer
 from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
-from common.method_wrappers.ServiceExceptions import AlreadyExistsException, InvalidArgumentException
-from common.proto.context_pb2 import (DeviceId, Empty, EndPointId, Link, LinkId, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum, TopologyId,ContextId,Topology
-    ,Device,DeviceDriverEnum, Uuid)
-from common.proto.pathcomp_pb2 import PathCompRequest
-from common.proto.service_pb2_grpc import ServiceServiceServicer
-from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string
+from common.proto.context_pb2 import DeviceId, Empty, EndPointId, Link, LinkId, Uuid
 from context.client.ContextClient import ContextClient
-from pathcomp.frontend.client.PathCompClient import PathCompClient
-from common.proto.bgpls_pb2 import (BgplsSpeaker,BgplsSpeakerId, DiscoveredDeviceList,DiscoveredDevice,
-                                    DiscoveredLink,DiscoveredLinkList,NodeDescriptors,BgplsSpeakerList)
+from common.proto.bgpls_pb2 import (
+    BgplsSpeaker, BgplsSpeakerId, DiscoveredDeviceList, DiscoveredDevice,
+    DiscoveredLink, DiscoveredLinkList, NodeDescriptors, BgplsSpeakerList
+)
 from common.proto.bgpls_pb2_grpc import BgplsServiceServicer
 
 def json_to_list(json_str : str) -> List[Union[str, Tuple[str, str]]]:

src/bgpls_speaker/service/__main__.py

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging, signal, sys, threading, time
+import logging, signal, sys, threading
 from prometheus_client import start_http_server
-from common.Constants import ServiceNameEnum
-from common.Settings import (
-    ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_log_level, get_metrics_port,
-    wait_for_environment_variables)
-from context.client.ContextClient import ContextClient
+from common.Settings import get_log_level, get_metrics_port
 from .tools.DiscoveredDBManager import DiscoveredDBManager
 from .BgplsService import BgplsService
-from .tools.JavaRunner import JavaRunner
 from .tools.GrpcServer import GrpcServer
 
 terminate = threading.Event()

src/bgpls_speaker/tests/test_unitary.py

+# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(logging.DEBUG)

src/device/Dockerfile

@@ -62,9 +62,11 @@ RUN python3 -m pip install -r requirements.txt
 
 # Add component files into working directory
 WORKDIR /var/teraflow
-COPY src/context/. context/
+COPY src/context/__init__.py context/__init__.py
+COPY src/context/client/. context/client/
+COPY src/monitoring/__init__.py monitoring/__init__.py
+COPY src/monitoring/client/. monitoring/client/
 COPY src/device/. device/
-COPY src/monitoring/. monitoring/
 
 RUN mkdir -p tests/tools/mock_ietf_actn_sdn_ctrl
 RUN touch tests/__init__.py

src/pathcomp/frontend/Dockerfile

@@ -54,26 +54,28 @@ RUN rm *.proto
 RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;
 
 # Create component sub-folders, get specific Python packages
-RUN mkdir -p /var/teraflow/pathcomp
-WORKDIR /var/teraflow/pathcomp
+RUN mkdir -p /var/teraflow/pathcomp/frontend
+WORKDIR /var/teraflow/pathcomp/frontend
 COPY src/pathcomp/frontend/requirements.in requirements.in
 RUN pip-compile --quiet --output-file=requirements.txt requirements.in
 RUN python3 -m pip install -r requirements.txt
 
 # Add component files into working directory
 WORKDIR /var/teraflow
+COPY src/pathcomp/__init__.py pathcomp/__init__.py
+COPY src/pathcomp/frontend/. pathcomp/frontend/
 COPY src/context/__init__.py context/__init__.py
 COPY src/context/client/. context/client/
 COPY src/device/__init__.py device/__init__.py
 COPY src/device/client/. device/client/
-COPY src/forecaster/. forecaster/
+COPY src/forecaster/__init__.py forecaster/__init__.py
+COPY src/forecaster/client/. forecaster/client/
 COPY src/monitoring/__init__.py monitoring/__init__.py
 COPY src/monitoring/client/. monitoring/client/
 COPY src/service/__init__.py service/__init__.py
 COPY src/service/client/. service/client/
 COPY src/slice/__init__.py slice/__init__.py
 COPY src/slice/client/. slice/client/
-COPY src/pathcomp/. pathcomp/
 
 # Start the service
 ENTRYPOINT ["python", "-m", "pathcomp.frontend.service"]

src/service/Dockerfile

@@ -62,7 +62,6 @@ RUN python3 -m pip install -r requirements.txt
 
 # Add component files into working directory
 WORKDIR /var/teraflow
-COPY src/service/. service/
 COPY src/context/__init__.py context/__init__.py
 COPY src/context/client/. context/client/
 COPY src/device/__init__.py device/__init__.py
@@ -71,6 +70,7 @@ COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py
 COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/
 COPY src/e2e_orchestrator/__init__.py e2e_orchestrator/__init__.py
 COPY src/e2e_orchestrator/client/. e2e_orchestrator/client/
+COPY src/service/. service/
 
 # Start the service
 ENTRYPOINT ["python", "-m", "service.service"]

src/slice/Dockerfile

@@ -62,10 +62,15 @@ RUN python3 -m pip install -r requirements.txt
 
 # Add component files into working directory
 WORKDIR /var/teraflow
-COPY src/context/. context/
-COPY src/interdomain/. interdomain/
-COPY src/pathcomp/. pathcomp/
-COPY src/service/. service/
+COPY src/context/__init__.py context/__init__.py
+COPY src/context/client/. context/client/
+COPY src/interdomain/__init__.py interdomain/__init__.py
+COPY src/interdomain/client/. interdomain/client/
+COPY src/pathcomp/__init__.py pathcomp/__init__.py
+COPY src/pathcomp/frontend/__init__.py pathcomp/frontend/__init__.py
+COPY src/pathcomp/frontend/client/. pathcomp/frontend/client/
+COPY src/service/__init__.py service/__init__.py
+COPY src/service/client/. service/client/
 COPY src/slice/. slice/
 
 # Start the service