Merge branch 'develop' of https://labs.etsi.org/rep/tfs/controller into feat/tid-webui-service-mgmt

.gitlab-ci.yml

@@ -31,7 +31,7 @@ include:
   - local: '/src/dbscanserving/.gitlab-ci.yml'
   - local: '/src/opticalattackmitigator/.gitlab-ci.yml'
   - local: '/src/opticalattackdetector/.gitlab-ci.yml'
-  - local: '/src/opticalattackmanager/.gitlab-ci.yml'
+  # - local: '/src/opticalattackmanager/.gitlab-ci.yml'
   - local: '/src/automation/.gitlab-ci.yml'
   - local: '/src/policy/.gitlab-ci.yml'
   #- local: '/src/webui/.gitlab-ci.yml'

src/dbscanserving/.gitlab-ci.yml

@@ -54,7 +54,7 @@ unit_test dbscanserving:
     - sleep 5
     - docker ps -a
     - docker logs $IMAGE_NAME
-    - docker exec ps -a
+    - docker ps -a
     - sleep 5
     - docker logs $IMAGE_NAME
     - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml"

src/dbscanserving/tests/test_unitary.py

@@ -23,30 +23,27 @@ from common.proto.dbscanserving_pb2 import (DetectionRequest,
                                             DetectionResponse, Sample)
 
 from dbscanserving.client.DbscanServingClient import DbscanServingClient
-from dbscanserving.Config import GRPC_SERVICE_PORT
 from dbscanserving.service.DbscanService import DbscanService
 
-port = 10000 + GRPC_SERVICE_PORT  # avoid privileged ports
-
 LOGGER = logging.getLogger(__name__)
 LOGGER.setLevel(logging.DEBUG)
 
 
 @pytest.fixture(scope="session")
 def dbscanserving_service():
-    _service = DbscanService(port=port)
+    _service = DbscanService()
     _service.start()
     yield _service
     _service.stop()
 
 
 @pytest.fixture(scope="session")
-def dbscanserving_client():
+def dbscanserving_client(dbscanserving_service: DbscanService):
     with patch.dict(
         os.environ,
         {
             "DBSCANSERVINGSERVICE_SERVICE_HOST": "127.0.0.1",
-            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(port),
+            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(dbscanserving_service.bind_port),
         },
         clear=True,
     ):

src/opticalattackdetector/.gitlab-ci.yml

@@ -49,9 +49,26 @@ unit_test opticalattackdetector:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
     - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
     - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
+    - if docker container ls | grep redis; then docker rm -f redis; else echo "redis image is not in the system"; fi
+    - if docker container ls | grep dbscanserving; then docker rm -f dbscanserving; else echo "dbscanserving image is not in the system"; fi
   script:
+    - export REDIS_PASSWORD=$(uuidgen)
+    - docker pull "redis:7.0-alpine"
+    - docker run --name redis -d --network=teraflowbridge -p 16379:6379 -e REDIS_PASSWORD=${REDIS_PASSWORD} --rm redis:7.0-alpine redis-server --requirepass ${REDIS_PASSWORD}
+    - docker logs redis
+    - REDIS_ADDRESS=$(docker inspect redis --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
+    - docker pull "$CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG"
+    - docker run --name dbscanserving -d -p 10008:10008 --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/dbscanserving:$IMAGE_TAG "python -m dbscanserving.service"
+    - docker logs dbscanserving
+    - DBSCANSERVING_ADDRESS=$(docker inspect dbscanserving --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
-    - docker run --name $IMAGE_NAME -d -p 10006:10006 --env-file "$PWD/src/$IMAGE_NAME/.env" -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+    - >
+      docker run --name $IMAGE_NAME -d -p 10006:10006 
+      -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" 
+      -e REDIS_PASSWORD=${REDIS_PASSWORD}
+      -e DBSCANSERVINGSERVICE_SERVICE_HOST=${DBSCANSERVING_ADDRESS}
+      -e CACHINGSERVICE_SERVICE_HOST=${REDIS_ADDRESS}
+      --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - sleep 5
     - docker ps -a
     - docker logs $IMAGE_NAME
@@ -59,6 +76,8 @@ unit_test opticalattackdetector:
   coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
   after_script:
     - docker rm -f $IMAGE_NAME
+    - docker rm -f redis
+    - docker rm -f dbscanserving
     - docker network rm teraflowbridge
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

src/opticalattackdetector/tests/test_unitary.py

@@ -13,88 +13,69 @@
 # limitations under the License.
 
 import logging
-import os
+import uuid
+import queue
 from unittest.mock import patch
 
 import pytest
-from common.Constants import (DEFAULT_GRPC_GRACE_PERIOD,
-                              DEFAULT_GRPC_MAX_WORKERS)
+
 from common.proto import dbscanserving_pb2 as dbscan
 from common.proto.optical_attack_detector_pb2 import DetectionRequest
+from common.tests.MockServicerImpl_Monitoring import MockServicerImpl_Monitoring
 
 from opticalattackdetector.client.OpticalAttackDetectorClient import \
     OpticalAttackDetectorClient
-from opticalattackdetector.Config import GRPC_SERVICE_PORT
 from opticalattackdetector.service.OpticalAttackDetectorService import \
     OpticalAttackDetectorService
 
 # from .example_objects import CONTEXT_ID, CONTEXT_ID_2, SERVICE_DEV1_DEV2
 
-port = 10000 + GRPC_SERVICE_PORT  # avoid privileged ports
-
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 
 @pytest.fixture(scope="session")
 def optical_attack_detector_service():
-    with patch.dict(
-        os.environ,
-        {
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC": str(port),
-            "DBSCANSERVINGSERVICE_SERVICE_HOST": "127.0.0.1",
-            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(port),
-        },
-        clear=True,
-    ):
-        _service = OpticalAttackDetectorService(
-            port=port,
-            max_workers=DEFAULT_GRPC_MAX_WORKERS,
-            grace_period=DEFAULT_GRPC_GRACE_PERIOD,
-        )
-        _service.start()
-        yield _service
-        _service.stop()
+    _service = OpticalAttackDetectorService()
+    _service.start()
+    yield _service
+    _service.stop()
 
 
 @pytest.fixture(scope="session")
-def optical_attack_detector_client(optical_attack_detector_service):
-    with patch.dict(
-        os.environ,
-        {
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC": str(port),
-            "DBSCANSERVINGSERVICE_SERVICE_HOST": "127.0.0.1",
-            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(port),
-        },
-        clear=True,
-    ):
-        _client = OpticalAttackDetectorClient()
-        yield _client
+def optical_attack_detector_client(optical_attack_detector_service: OpticalAttackDetectorService):
+    _client = OpticalAttackDetectorClient(
+        host=optical_attack_detector_service.bind_address,
+        port=optical_attack_detector_service.bind_port,
+    )
+    yield _client
     _client.close()
 
 
 def test_detect_attack(
+    optical_attack_detector_service: OpticalAttackDetectorService,
     optical_attack_detector_client: OpticalAttackDetectorClient,
 ):
     message = dbscan.DetectionResponse()
     message.cluster_indices.extend([0, 1, -1, -1, -1])
+
+    monitoring_mock = MockServicerImpl_Monitoring(queue_samples = queue.Queue())
     with patch(
         "opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.attack_mitigator_client"
     ) as mitigator, patch(
-        "opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.monitoring_client"
+        "opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.monitoring_client",
+        monitoring_mock,
     ) as monitoring, patch(
         "opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.dbscanserving_client.Detect",
         # TODO: return dumb object with "cluster_indices" attribute
         # idea: create new response object
         return_value=message,
     ) as dbscanserving:
-        request: DetectionRequest = DetectionRequest()
-        request.service_id.context_id.context_uuid.uuid = ""
-        request.service_id.service_uuid.uuid = str("")
-        request.kpi_id.kpi_id.uuid = ""
-        optical_attack_detector_client.DetectAttack(request)
-        mitigator.NotifyAttack.assert_called_once()
-        monitoring.IncludeKpi.assert_called_once()
-        dbscanserving.assert_called_once()
+        for _ in range(10):
+            request: DetectionRequest = DetectionRequest()
+            request.service_id.context_id.context_uuid.uuid = str(uuid.uuid4())
+            request.service_id.service_uuid.uuid = str(uuid.uuid4())
+            request.kpi_id.kpi_id.uuid = "1"
+            optical_attack_detector_client.DetectAttack(request)
+            dbscanserving.assert_called_once()
+            monitoring.IncludeKpi.assert_called_once()
+        mitigator.NotifyAttack.assert_called()

src/opticalattackmanager/.gitlab-ci.yml

@@ -52,11 +52,11 @@ unit_test opticalattackmanager:
     - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
   script:
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
-    - docker run --name $IMAGE_NAME -d -p 10005:10005 -v "$PWD/src/$IMAGE_NAME/tests:/home/teraflow/$IMAGE_NAME/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+    - docker run --name $IMAGE_NAME -d -p 10005:10005 -e LOG_LEVEL=DEBUG -v "$PWD/src/$IMAGE_NAME/tests:/home/teraflow/controller/$IMAGE_NAME/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - docker ps -a
     - docker logs $IMAGE_NAME
-    - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/teraflow/$IMAGE_NAME/results/${IMAGE_NAME}_report.xml"
-    - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
+    - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/teraflow/controller/$IMAGE_NAME/results/${IMAGE_NAME}_report.xml; coverage report --include='${IMAGE_NAME}/*' --show-missing"
+    - ls -la src/$IMAGE_NAME/tests
   coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
   after_script:
     - docker rm -f $IMAGE_NAME
@@ -74,9 +74,9 @@ unit_test opticalattackmanager:
       - manifests/${IMAGE_NAME}service.yaml
       - .gitlab-ci.yml
   artifacts:
-      when: always
-      reports:
-        junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
+    when: always
+    reports:
+      junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
 
 # Deployment of the service in Kubernetes Cluster
 # deploy opticalattackmanager:

src/opticalattackmanager/tests/test_unitary.py

@@ -13,108 +13,27 @@
 # limitations under the License.
 
 import logging
-import os
-from unittest.mock import patch
+import uuid
 
-import pytest
+from opticalattackmanager.utils.monitor import delegate_services
 
-from common.Constants import (DEFAULT_GRPC_GRACE_PERIOD,
-                              DEFAULT_GRPC_MAX_WORKERS,
-                              DEFAULT_SERVICE_GRPC_PORTS, ServiceNameEnum)
-from common.proto import dbscanserving_pb2 as dbscan
-from common.proto.optical_attack_detector_pb2 import DetectionRequest
-from opticalattackdetector.client.OpticalAttackDetectorClient import \
-    OpticalAttackDetectorClient
-from opticalattackdetector.Config import GRPC_SERVICE_PORT
-from opticalattackdetector.service.OpticalAttackDetectorService import \
-    OpticalAttackDetectorService
-
-# from .example_objects import CONTEXT_ID, CONTEXT_ID_2, SERVICE_DEV1_DEV2
-
-port = 10000 + GRPC_SERVICE_PORT  # avoid privileged ports
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 
-@pytest.fixture(scope="session")
-def optical_attack_detector_service():
-    with patch.dict(
-        os.environ,
+def test_delegate_services():
+    service_list = [
         {
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(
-                    ServiceNameEnum.OPTICALATTACKDETECTOR.value
-                )
-            ),
-            "OPTICALATTACKMITIGATORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKMITIGATORSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(
-                    ServiceNameEnum.OPTICALATTACKMITIGATOR.value
-                )
-            ),
-            "DBSCANSERVINGSERVICE_SERVICE_HOST": "127.0.0.1",
-            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(ServiceNameEnum.DBSCANSERVING.value)
-            ),
-        },
-        clear=True,
-    ):
-        _service = OpticalAttackDetectorService(
-            port=port,
-            max_workers=DEFAULT_GRPC_MAX_WORKERS,
-            grace_period=DEFAULT_GRPC_GRACE_PERIOD,
-        )
-        # mocker_context_client = mock.patch('opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.context_client')
-        # mocker_context_client.start()
-
-        # mocker_influx_db = mock.patch('opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.influxdb_client')
-        # mocker_influx_db.start()
-
-        _service.start()
-        yield _service
-        _service.stop()
-        # mocker_context_client.stop()
-        # mocker_influx_db.stop()
-
-
-@pytest.fixture(scope="session")
-def optical_attack_detector_client(optical_attack_detector_service):
-    with patch.dict(
-        os.environ,
-        {
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(
-                    ServiceNameEnum.OPTICALATTACKDETECTOR.value
-                )
-            ),
-            "OPTICALATTACKMITIGATORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKMITIGATORSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(
-                    ServiceNameEnum.OPTICALATTACKMITIGATOR.value
-                )
-            ),
-            "DBSCANSERVINGSERVICE_SERVICE_HOST": "127.0.0.1",
-            "DBSCANSERVINGSERVICE_SERVICE_PORT_GRPC": str(
-                1000
-                + DEFAULT_SERVICE_GRPC_PORTS.get(ServiceNameEnum.DBSCANSERVING.value)
-            ),
-        },
-        clear=True,
-    ):
-        _client = OpticalAttackDetectorClient()
-        yield _client
-    _client.close()
-
-
-def test_detect_attack(
-    optical_attack_detector_client: OpticalAttackDetectorClient,
-):
-    LOGGER.info("placeholder")
+            "context": uuid.uuid4(),
+            "service": uuid.uuid4(),
+            "kpi": 10,
+        }
+        for _ in range(10)]
+    delegate_services(
+        service_list=service_list,
+        start_index=0,
+        end_index=9,
+        host="127.0.0.1",
+        port="10006",
+        monitoring_interval=10,
+    )

src/opticalattackmanager/utils/monitor.py

@@ -17,7 +17,6 @@ import logging
 import traceback
 
 from grpclib.client import Channel
-from prometheus_client import Counter
 
 from common.proto.asyncio.optical_attack_detector_grpc import \
     OpticalAttackDetectorServiceStub

src/opticalattackmitigator/.gitlab-ci.yml

@@ -33,7 +33,6 @@ build opticalattackmitigator:
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py
-      - src/$IMAGE_NAME/tests/Dockerfile
       - manifests/${IMAGE_NAME}service.yaml
       - .gitlab-ci.yml
 
@@ -48,7 +47,7 @@ unit_test opticalattackmitigator:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
     - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
-    - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
+    - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi
   script:
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
     - docker run --name $IMAGE_NAME -d -p 10007:10007 -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
@@ -66,8 +65,7 @@ unit_test opticalattackmitigator:
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py
-      - src/$IMAGE_NAME/tests/Dockerfile
-      - manifests/$IMAGE_NAMEservice.yaml
+      - manifests/${IMAGE_NAME}service.yaml
       - .gitlab-ci.yml
   artifacts:
     when: always

src/opticalattackmitigator/tests/test_unitary.py

@@ -17,29 +17,20 @@ import os
 from unittest.mock import patch
 
 import pytest
-from common.Constants import (DEFAULT_GRPC_GRACE_PERIOD,
-                              DEFAULT_GRPC_MAX_WORKERS)
 from common.proto.optical_attack_mitigator_pb2 import AttackDescription
 
 from opticalattackmitigator.client.OpticalAttackMitigatorClient import \
     OpticalAttackMitigatorClient
-from opticalattackmitigator.Config import GRPC_SERVICE_PORT
 from opticalattackmitigator.service.OpticalAttackMitigatorService import \
     OpticalAttackMitigatorService
 
-port = 10000 + GRPC_SERVICE_PORT  # avoid privileged ports
-
 LOGGER = logging.getLogger(__name__)
 LOGGER.setLevel(logging.DEBUG)
 
 
 @pytest.fixture(scope="session")
 def optical_attack_mitigator_service():
-    _service = OpticalAttackMitigatorService(
-        port=port,
-        max_workers=DEFAULT_GRPC_MAX_WORKERS,
-        grace_period=DEFAULT_GRPC_GRACE_PERIOD,
-    )
+    _service = OpticalAttackMitigatorService()
     _service.start()
     yield _service
     _service.stop()
@@ -51,7 +42,7 @@ def optical_attack_mitigator_client(optical_attack_mitigator_service):
         os.environ,
         {
             "OPTICALATTACKMITIGATORSERVICE_SERVICE_HOST": "127.0.0.1",
-            "OPTICALATTACKMITIGATORSERVICE_SERVICE_PORT_GRPC": str(port),
+            "OPTICALATTACKMITIGATORSERVICE_SERVICE_PORT_GRPC": str(optical_attack_mitigator_service.bind_port),
         },
         clear=True,
     ):

src/tests/scenario3/optical/ofc23/run_experiment_demo.py

@@ -21,8 +21,7 @@ import threading
 import redis
 from kubernetes import client, config
 
-from common.Constants import ServiceNameEnum
-from common.Settings import get_service_host, get_setting, wait_for_environment_variables
+from common.Settings import get_setting, wait_for_environment_variables
 
 # Configs can be set in Configuration class directly or using helper utility
 namespace = get_setting("TFS_K8S_NAMESPACE", default="tfs")