Intermediate commit of work in progress.

.gitignore

@@ -53,6 +53,7 @@ coverage.xml
 .pytest_cache/
 .benchmarks/
 cover/
+*_report.xml
 
 # Translations
 *.mo

manifests/monitoringservice.yaml

@@ -40,12 +40,12 @@ spec:
           value: "1000"
         - name: QDB_CAIRO_MAX_UNCOMMITTED_ROWS
           value: "100000"
-        readinessProbe:
-          exec:
-            command: ["curl", "-XGET", "localhost:9000"]
-        livenessProbe:
-          exec:
-            command: ["curl", "-XGET", "localhost:9003/metrics"]
+        # readinessProbe:
+        #   exec:
+        #     command: ["curl", "-XGET", "localhost:9000"]
+        # livenessProbe:
+        #   exec:
+        #     command: ["curl", "-XGET", "localhost:9003/metrics"]
         resources:
           requests:
             cpu: 250m

manifests/opticalcentralizedattackdetectorservice.yaml → manifests/opticalattackdetectorservice.yaml

@@ -15,20 +15,20 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: opticalcentralizedattackdetectorservice
+  name: opticalattackdetectorservice
 spec:
   selector:
     matchLabels:
-      app: opticalcentralizedattackdetectorservice
+      app: opticalattackdetectorservice
   template:
     metadata:
       labels:
-        app: opticalcentralizedattackdetectorservice
+        app: opticalattackdetectorservice
     spec:
       terminationGracePeriodSeconds: 5
       containers:
       - name: server
-        image: registry.gitlab.com/teraflow-h2020/controller/opticalcentralizedattackdetector:latest
+        image: registry.gitlab.com/teraflow-h2020/controller/opticalattackdetector:latest
         imagePullPolicy: Always
         ports:
         - containerPort: 10005
@@ -52,11 +52,11 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: opticalcentralizedattackdetectorservice
+  name: opticalattackdetectorservice
 spec:
   type: ClusterIP
   selector:
-    app: opticalcentralizedattackdetectorservice
+    app: opticalattackdetectorservice
   ports:
   - name: grpc
     port: 10005

my_deploy.sh

@@ -7,7 +7,7 @@ export TFS_REGISTRY_IMAGE=""  # http://localhost:32000/tfs/
 #   interdomain slice pathcomp dlt
 #   dbscanserving opticalattackmitigator opticalcentralizedattackdetector
 #   l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector
-export TFS_COMPONENTS="context device automation service compute monitoring webui dbscanserving opticalattackmitigator opticalcentralizedattackdetector"
+export TFS_COMPONENTS="context device automation service compute monitoring webui dbscanserving opticalattackmitigator opticalattackdetector"
 
 # Set the tag you want to use for your images.
 export TFS_IMAGE_TAG="dev"

proto/optical_attack_detector.proto

@@ -20,7 +20,6 @@ import "context.proto";
 import "monitoring.proto";
 
 service OpticalAttackDetectorService {
-  rpc NotifyServiceUpdate (context.Service   ) returns (context.Empty) {}
   
   // rpc that triggers the attack detection loop
   rpc DetectAttack        (context.Empty     ) returns (context.Empty) {}

run_tests_docker.sh

+#!/bin/bash
+
+########################################################################################################################
+# Define your deployment settings here
+########################################################################################################################
+
+# Set the URL of your local Docker registry where the images will be uploaded to. Leave it blank if you do not want to
+# use any Docker registry.
+REGISTRY_IMAGE=""
+#REGISTRY_IMAGE="http://my-container-registry.local/"
+
+# Set the list of components you want to build images for, and deploy.
+COMPONENTS="context device automation policy service compute monitoring centralizedattackdetector"
+
+# Set the tag you want to use for your images.
+IMAGE_TAG="tf-dev"
+
+# Constants
+TMP_FOLDER="./tmp"
+
+TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
+mkdir -p $TMP_LOGS_FOLDER
+
+for COMPONENT in $COMPONENTS; do
+    echo "Processing '$COMPONENT' component..."
+    IMAGE_NAME="$COMPONENT:$IMAGE_TAG"
+    IMAGE_URL="$REGISTRY_IMAGE/$IMAGE_NAME"
+
+    echo "  Building Docker image..."
+    BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
+
+    if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then
+        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
+    else 
+        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/ > "$BUILD_LOG"
+    fi
+
+    if [ -n "$REGISTRY_IMAGE" ]; then
+        echo "Pushing Docker image to '$REGISTRY_IMAGE'..."
+
+        TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
+        docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"
+
+        PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
+        docker push "$IMAGE_URL" > "$PUSH_LOG"
+    fi
+done
+
+echo "Preparing for running the tests..."
+
+if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi  
+
+for COMPONENT in $COMPONENTS; do
+    IMAGE_NAME="$COMPONENT:$IMAGE_TAG"
+    echo "  Running tests for $COMPONENT:"
+    docker run -it -d --name $COMPONENT $IMAGE_NAME --network=teraflowbridge
+    docker exec -it $COMPONENT bash -c "pytest --log-level=DEBUG --verbose $COMPONENT/tests/test_unitary.py"
+    docker stop $COMPONENT
+done

scripts/build_run_report_tests.sh → scripts/build_run_report_tests_locally.sh

@@ -40,7 +40,7 @@ docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$COMPONENT_NAME/Dockerfile .
 
 docker run --name $IMAGE_NAME -d -v "${PWD}/src/${COMPONENT_NAME}/tests:/home/${COMPONENT_NAME}/results" --network=teraflowbridge --rm $IMAGE_NAME:$IMAGE_TAG
 
-docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $COMPONENT_NAME/tests/test_unitary.py --junitxml=/home/${COMPONENT_NAME}/results/${COMPONENT_NAME}_report.xml"
+docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $COMPONENT_NAME/tests/ --junitxml=/home/${COMPONENT_NAME}/results/${COMPONENT_NAME}_report.xml"
 
 PROJECTDIR=`pwd`
 

src/dbscanserving/tests/dbscanserving_report.xml

-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="3" time="2.030" timestamp="2022-07-28T15:49:58.569972" hostname="799d7fffaad2"><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_correct" time="0.426" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_incorrect" time="0.288" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_clusters" time="0.324" /></testsuite></testsuites>
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="3" time="2.094" timestamp="2022-09-05T12:25:16.641312" hostname="d13125c46af6"><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_correct" time="0.444" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_incorrect" time="0.273" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_clusters" time="0.319" /></testsuite></testsuites>
\ No newline at end of file

src/opticalattackdetector/Config.py

@@ -19,11 +19,9 @@ LOG_LEVEL = logging.DEBUG
 
 # gRPC settings
 GRPC_SERVICE_PORT = 10005
-GRPC_MAX_WORKERS  = 10
-GRPC_GRACE_PERIOD = 60
 
 # service settings
-MONITORING_INTERVAL = 2  # monitoring interval in seconds
+MONITORING_INTERVAL = 10  # monitoring interval in seconds
 #TODO: adjust the addresses below for the specific case
 MONITORING_SERVICE_ADDRESS = 'monitoringservice'  # address/name of the monitoring service
 # MONITORING_SERVICE_ADDRESS = '10.99.41.20'  # address/name of the monitoring service

src/opticalattackdetector/service/OpticalAttackDetectorService.py

@@ -15,22 +15,24 @@
 import grpc
 import logging
 from concurrent import futures
+
 from grpc_health.v1.health import HealthServicer, OVERALL_HEALTH
 from grpc_health.v1.health_pb2 import HealthCheckResponse
 from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
+
+from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD
 from common.proto.optical_attack_detector_pb2_grpc import (
     add_OpticalAttackDetectorServiceServicer_to_server)
 from opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl import (
     OpticalAttackDetectorServiceServicerImpl)
-from opticalattackdetector.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
+from opticalattackdetector.Config import GRPC_SERVICE_PORT
 
-BIND_ADDRESS = '0.0.0.0'
 LOGGER = logging.getLogger(__name__)
 
 class OpticalAttackDetectorService:
     def __init__(
-        self, address=BIND_ADDRESS, port=GRPC_SERVICE_PORT, max_workers=GRPC_MAX_WORKERS,
-        grace_period=GRPC_GRACE_PERIOD):
+        self, address=DEFAULT_GRPC_BIND_ADDRESS, port=GRPC_SERVICE_PORT, max_workers=DEFAULT_GRPC_MAX_WORKERS,
+        grace_period=DEFAULT_GRPC_GRACE_PERIOD):
 
         self.address = address
         self.port = port

src/opticalattackdetector/service/OpticalAttackDetectorServiceServicerImpl.py

@@ -63,10 +63,6 @@ class OpticalAttackDetectorServiceServicerImpl(OpticalAttackDetectorServiceServi
         LOGGER.debug('Creating Servicer...')
         LOGGER.debug('Servicer Created')
 
-    @safe_and_metered_rpc_method(METRICS, LOGGER)
-    def NotifyServiceUpdate(self, request : Service, context : grpc.ServicerContext) -> Empty:
-        return Empty()
-
     @safe_and_metered_rpc_method(METRICS, LOGGER)
     def DetectAttack(self, request : Empty, context : grpc.ServicerContext) -> Empty:
         

src/opticalattackdetector/service/__main__.py

@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os, logging, signal, sys, time, threading, multiprocessing
+import logging, signal, sys, time, threading
 from prometheus_client import start_http_server
 
+from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD
 from common.Settings import get_log_level, get_metrics_port, get_setting
 from opticalattackdetector.Config import (
-    GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD, MONITORING_INTERVAL)
+    GRPC_SERVICE_PORT, MONITORING_INTERVAL)
 from common.proto.context_pb2 import (Empty,
     Context,  ContextId,  ContextIdList,  ContextList,
     Service,  ServiceId,  ServiceIdList,  ServiceList
@@ -57,8 +58,8 @@ def main():
     LOGGER = logging.getLogger(__name__)
 
     service_port = get_setting('OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC', default=GRPC_SERVICE_PORT)
-    max_workers  = get_setting('MAX_WORKERS',                                    default=GRPC_MAX_WORKERS )
-    grace_period = get_setting('GRACE_PERIOD',                                   default=GRPC_GRACE_PERIOD)
+    max_workers  = get_setting('MAX_WORKERS',                                    default=DEFAULT_GRPC_MAX_WORKERS )
+    grace_period = get_setting('GRACE_PERIOD',                                   default=DEFAULT_GRPC_GRACE_PERIOD)
     monitoring_interval = get_setting('MONITORING_INTERVAL',                     default=MONITORING_INTERVAL     )
 
     signal.signal(signal.SIGINT,  signal_handler)

src/opticalattackdetector/tests/test_unitary.py

@@ -14,10 +14,13 @@
 
 import logging, pytest
 from unittest.mock import patch
-from opticalattackdetector.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
-from opticalattackdetector.client.OpticalAttackDetectorClient import OpticalAttackDetectorClient
+
+from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD
 from common.proto.context_pb2 import ContextIdList, ContextId, Empty, Service, ContextId, ServiceList
 from common.proto.monitoring_pb2 import Kpi, KpiList
+
+from opticalattackdetector.Config import GRPC_SERVICE_PORT
+from opticalattackdetector.client.OpticalAttackDetectorClient import OpticalAttackDetectorClient
 from opticalattackdetector.service.OpticalAttackDetectorService import OpticalAttackDetectorService
 from .example_objects import CONTEXT_ID, CONTEXT_ID_2, SERVICE_DEV1_DEV2
 
@@ -29,7 +32,7 @@ LOGGER.setLevel(logging.DEBUG)
 @pytest.fixture(scope='session')
 def optical_attack_detector_service():
     _service = OpticalAttackDetectorService(
-        port=port, max_workers=GRPC_MAX_WORKERS, grace_period=GRPC_GRACE_PERIOD)
+        port=port, max_workers=DEFAULT_GRPC_MAX_WORKERS, grace_period=DEFAULT_GRPC_GRACE_PERIOD)
     # mocker_context_client = mock.patch('opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.context_client')
     # mocker_context_client.start()
 

src/opticalattackmitigator/tests/opticalattackmitigator_report.xml

-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="1" time="0.295" timestamp="2022-07-28T16:09:47.894890" hostname="35c685c0deaa"><testcase classname="opticalattackmitigator.tests.test_unitary" name="test_call_service" time="0.012" /></testsuite></testsuites>
\ No newline at end of file

tutorial/3-2-develop-cth.md

 # 3.2. Development Commands, Tricks, and Hints (WORK IN PROGRESS)
 
-## Building, running, testing and reporting code coverage locally
 
-The project runs a CI/CD loops that ensures that all tests are run whenever new code is committed to our reporitory.
-However, committing and waiting for the pipeline to run can take substantial time.
-For this reason, we prepared a script that runs in your local machine, builds the container image and executes the tests within the image.
-
-To use the script receives one argument that is the name of the component you want to run.
-For instance, if you want to build and run the tests of the `device` component, you can run:
-
-```shell
-scripts/build_run_report_tests.sh device
-```
 
 ## Items to be addressed:
 - pytest flags: --log-level=INFO --verbose -o log_cli=true -o log_file=my_log_file.log -o log_file_level=DEBUG