diff --git a/.gitignore b/.gitignore index d4a32e40f0f518a7c4349bae9732ffce0d49bdf7..6d143f0e9d3d44331783873d4eb6bc9249ce8aaf 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ coverage.xml .pytest_cache/ .benchmarks/ cover/ +*_report.xml # Translations *.mo diff --git a/manifests/monitoringservice.yaml b/manifests/monitoringservice.yaml index 7f0bee9efc68e66c72487624241e763dccb2fc76..19ce31d11fbf4c4faf56ff76a54d36b1b300b0cf 100644 --- a/manifests/monitoringservice.yaml +++ b/manifests/monitoringservice.yaml @@ -40,12 +40,12 @@ spec: value: "1000" - name: QDB_CAIRO_MAX_UNCOMMITTED_ROWS value: "100000" - readinessProbe: - exec: - command: ["curl", "-XGET", "localhost:9000"] - livenessProbe: - exec: - command: ["curl", "-XGET", "localhost:9003/metrics"] + # readinessProbe: + # exec: + # command: ["curl", "-XGET", "localhost:9000"] + # livenessProbe: + # exec: + # command: ["curl", "-XGET", "localhost:9003/metrics"] resources: requests: cpu: 250m diff --git a/manifests/opticalcentralizedattackdetectorservice.yaml b/manifests/opticalattackdetectorservice.yaml similarity index 84% rename from manifests/opticalcentralizedattackdetectorservice.yaml rename to manifests/opticalattackdetectorservice.yaml index 664bcb54348e533ff40c7f882b5668f727a39053..f19953df7b58b2ccbce725d1c4532dd65e7f4e11 100644 --- a/manifests/opticalcentralizedattackdetectorservice.yaml +++ b/manifests/opticalattackdetectorservice.yaml @@ -15,20 +15,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: opticalcentralizedattackdetectorservice + name: opticalattackdetectorservice spec: selector: matchLabels: - app: opticalcentralizedattackdetectorservice + app: opticalattackdetectorservice template: metadata: labels: - app: opticalcentralizedattackdetectorservice + app: opticalattackdetectorservice spec: terminationGracePeriodSeconds: 5 containers: - name: server - image: registry.gitlab.com/teraflow-h2020/controller/opticalcentralizedattackdetector:latest + image: registry.gitlab.com/teraflow-h2020/controller/opticalattackdetector:latest imagePullPolicy: Always ports: - containerPort: 10005 @@ -52,11 +52,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: opticalcentralizedattackdetectorservice + name: opticalattackdetectorservice spec: type: ClusterIP selector: - app: opticalcentralizedattackdetectorservice + app: opticalattackdetectorservice ports: - name: grpc port: 10005 diff --git a/my_deploy.sh b/my_deploy.sh index 9071a91f6f5064ed482603fbdfc8bae59a010ba1..9d8e6e9dbf6b4cc77c1edee9141383f2bed3155e 100644 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -7,7 +7,7 @@ export TFS_REGISTRY_IMAGE="" # http://localhost:32000/tfs/ # interdomain slice pathcomp dlt # dbscanserving opticalattackmitigator opticalcentralizedattackdetector # l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector -export TFS_COMPONENTS="context device automation service compute monitoring webui dbscanserving opticalattackmitigator opticalcentralizedattackdetector" +export TFS_COMPONENTS="context device automation service compute monitoring webui dbscanserving opticalattackmitigator opticalattackdetector" # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/proto/optical_attack_detector.proto b/proto/optical_attack_detector.proto index 9e4176f12a378651a6793e4e852e196b4b4b31b4..afc8c558587544ce8d361296bcc0ec5923e1b098 100644 --- a/proto/optical_attack_detector.proto +++ b/proto/optical_attack_detector.proto @@ -20,7 +20,6 @@ import "context.proto"; import "monitoring.proto"; service OpticalAttackDetectorService { - rpc NotifyServiceUpdate (context.Service ) returns (context.Empty) {} // rpc that triggers the attack detection loop rpc DetectAttack (context.Empty ) returns (context.Empty) {} diff --git a/run_tests_docker.sh b/run_tests_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..fd885140999ac0f045c162f361f0075af96a8d48 --- /dev/null +++ b/run_tests_docker.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# Set the URL of your local Docker registry where the images will be uploaded to. Leave it blank if you do not want to +# use any Docker registry. +REGISTRY_IMAGE="" +#REGISTRY_IMAGE="http://my-container-registry.local/" + +# Set the list of components you want to build images for, and deploy. +COMPONENTS="context device automation policy service compute monitoring centralizedattackdetector" + +# Set the tag you want to use for your images. +IMAGE_TAG="tf-dev" + +# Constants +TMP_FOLDER="./tmp" + +TMP_LOGS_FOLDER="$TMP_FOLDER/logs" +mkdir -p $TMP_LOGS_FOLDER + +for COMPONENT in $COMPONENTS; do + echo "Processing '$COMPONENT' component..." + IMAGE_NAME="$COMPONENT:$IMAGE_TAG" + IMAGE_URL="$REGISTRY_IMAGE/$IMAGE_NAME" + + echo " Building Docker image..." + BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" + + if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then + docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" + else + docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/ > "$BUILD_LOG" + fi + + if [ -n "$REGISTRY_IMAGE" ]; then + echo "Pushing Docker image to '$REGISTRY_IMAGE'..." + + TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" + docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" + + PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" + docker push "$IMAGE_URL" > "$PUSH_LOG" + fi +done + +echo "Preparing for running the tests..." + +if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + +for COMPONENT in $COMPONENTS; do + IMAGE_NAME="$COMPONENT:$IMAGE_TAG" + echo " Running tests for $COMPONENT:" + docker run -it -d --name $COMPONENT $IMAGE_NAME --network=teraflowbridge + docker exec -it $COMPONENT bash -c "pytest --log-level=DEBUG --verbose $COMPONENT/tests/test_unitary.py" + docker stop $COMPONENT +done diff --git a/scripts/build_run_report_tests.sh b/scripts/build_run_report_tests_locally.sh similarity index 92% rename from scripts/build_run_report_tests.sh rename to scripts/build_run_report_tests_locally.sh index 79b1cc77c7e74ca8d449b42a2b84f0d53207199d..9bdc81d9894df35a6bcc325d78e7f1f5214e8a96 100755 --- a/scripts/build_run_report_tests.sh +++ b/scripts/build_run_report_tests_locally.sh @@ -40,7 +40,7 @@ docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$COMPONENT_NAME/Dockerfile . docker run --name $IMAGE_NAME -d -v "${PWD}/src/${COMPONENT_NAME}/tests:/home/${COMPONENT_NAME}/results" --network=teraflowbridge --rm $IMAGE_NAME:$IMAGE_TAG -docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $COMPONENT_NAME/tests/test_unitary.py --junitxml=/home/${COMPONENT_NAME}/results/${COMPONENT_NAME}_report.xml" +docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $COMPONENT_NAME/tests/ --junitxml=/home/${COMPONENT_NAME}/results/${COMPONENT_NAME}_report.xml" PROJECTDIR=`pwd` diff --git a/src/dbscanserving/tests/dbscanserving_report.xml b/src/dbscanserving/tests/dbscanserving_report.xml index 3c8246fa003e58554c915ad2a9b5dec40304367a..05e3dc8ac3a54c00eeff70152b4d42cbcfe1c7e0 100644 --- a/src/dbscanserving/tests/dbscanserving_report.xml +++ b/src/dbscanserving/tests/dbscanserving_report.xml @@ -1 +1 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="3" time="2.030" timestamp="2022-07-28T15:49:58.569972" hostname="799d7fffaad2"><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_correct" time="0.426" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_incorrect" time="0.288" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_clusters" time="0.324" /></testsuite></testsuites> \ No newline at end of file +<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="3" time="2.094" timestamp="2022-09-05T12:25:16.641312" hostname="d13125c46af6"><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_correct" time="0.444" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_incorrect" time="0.273" /><testcase classname="dbscanserving.tests.test_unitary" name="test_detection_clusters" time="0.319" /></testsuite></testsuites> \ No newline at end of file diff --git a/src/opticalattackdetector/Config.py b/src/opticalattackdetector/Config.py index 5fd3e97e9f590bea9cdfc6ebf8e93e90fb0a4fea..f959cbfd40b57bb01fe2fdd183ab94607449bacf 100644 --- a/src/opticalattackdetector/Config.py +++ b/src/opticalattackdetector/Config.py @@ -19,11 +19,9 @@ LOG_LEVEL = logging.DEBUG # gRPC settings GRPC_SERVICE_PORT = 10005 -GRPC_MAX_WORKERS = 10 -GRPC_GRACE_PERIOD = 60 # service settings -MONITORING_INTERVAL = 2 # monitoring interval in seconds +MONITORING_INTERVAL = 10 # monitoring interval in seconds #TODO: adjust the addresses below for the specific case MONITORING_SERVICE_ADDRESS = 'monitoringservice' # address/name of the monitoring service # MONITORING_SERVICE_ADDRESS = '10.99.41.20' # address/name of the monitoring service diff --git a/src/opticalattackdetector/service/OpticalAttackDetectorService.py b/src/opticalattackdetector/service/OpticalAttackDetectorService.py index 357ef2ab8eef00ce894808634be7eca85f26b22d..9bc1cc29758037b04ec2580fe94c0ec0c7e56f32 100644 --- a/src/opticalattackdetector/service/OpticalAttackDetectorService.py +++ b/src/opticalattackdetector/service/OpticalAttackDetectorService.py @@ -15,22 +15,24 @@ import grpc import logging from concurrent import futures + from grpc_health.v1.health import HealthServicer, OVERALL_HEALTH from grpc_health.v1.health_pb2 import HealthCheckResponse from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server + +from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD from common.proto.optical_attack_detector_pb2_grpc import ( add_OpticalAttackDetectorServiceServicer_to_server) from opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl import ( OpticalAttackDetectorServiceServicerImpl) -from opticalattackdetector.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD +from opticalattackdetector.Config import GRPC_SERVICE_PORT -BIND_ADDRESS = '0.0.0.0' LOGGER = logging.getLogger(__name__) class OpticalAttackDetectorService: def __init__( - self, address=BIND_ADDRESS, port=GRPC_SERVICE_PORT, max_workers=GRPC_MAX_WORKERS, - grace_period=GRPC_GRACE_PERIOD): + self, address=DEFAULT_GRPC_BIND_ADDRESS, port=GRPC_SERVICE_PORT, max_workers=DEFAULT_GRPC_MAX_WORKERS, + grace_period=DEFAULT_GRPC_GRACE_PERIOD): self.address = address self.port = port diff --git a/src/opticalattackdetector/service/OpticalAttackDetectorServiceServicerImpl.py b/src/opticalattackdetector/service/OpticalAttackDetectorServiceServicerImpl.py index d6fd1787bf93222457eb44f3bb7ee7d649c776c7..73c5c6abfb0b656b33469c1ba486c3add68d9d56 100644 --- a/src/opticalattackdetector/service/OpticalAttackDetectorServiceServicerImpl.py +++ b/src/opticalattackdetector/service/OpticalAttackDetectorServiceServicerImpl.py @@ -63,10 +63,6 @@ class OpticalAttackDetectorServiceServicerImpl(OpticalAttackDetectorServiceServi LOGGER.debug('Creating Servicer...') LOGGER.debug('Servicer Created') - @safe_and_metered_rpc_method(METRICS, LOGGER) - def NotifyServiceUpdate(self, request : Service, context : grpc.ServicerContext) -> Empty: - return Empty() - @safe_and_metered_rpc_method(METRICS, LOGGER) def DetectAttack(self, request : Empty, context : grpc.ServicerContext) -> Empty: diff --git a/src/opticalattackdetector/service/__main__.py b/src/opticalattackdetector/service/__main__.py index c57a75e8dc47f1a23b387492054cee83fbfb3189..4b7e9627af35d8e2048a96d22721d325c2d027d2 100644 --- a/src/opticalattackdetector/service/__main__.py +++ b/src/opticalattackdetector/service/__main__.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, logging, signal, sys, time, threading, multiprocessing +import logging, signal, sys, time, threading from prometheus_client import start_http_server +from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD from common.Settings import get_log_level, get_metrics_port, get_setting from opticalattackdetector.Config import ( - GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD, MONITORING_INTERVAL) + GRPC_SERVICE_PORT, MONITORING_INTERVAL) from common.proto.context_pb2 import (Empty, Context, ContextId, ContextIdList, ContextList, Service, ServiceId, ServiceIdList, ServiceList @@ -57,8 +58,8 @@ def main(): LOGGER = logging.getLogger(__name__) service_port = get_setting('OPTICALATTACKDETECTORSERVICE_SERVICE_PORT_GRPC', default=GRPC_SERVICE_PORT) - max_workers = get_setting('MAX_WORKERS', default=GRPC_MAX_WORKERS ) - grace_period = get_setting('GRACE_PERIOD', default=GRPC_GRACE_PERIOD) + max_workers = get_setting('MAX_WORKERS', default=DEFAULT_GRPC_MAX_WORKERS ) + grace_period = get_setting('GRACE_PERIOD', default=DEFAULT_GRPC_GRACE_PERIOD) monitoring_interval = get_setting('MONITORING_INTERVAL', default=MONITORING_INTERVAL ) signal.signal(signal.SIGINT, signal_handler) diff --git a/src/opticalattackdetector/tests/test_unitary.py b/src/opticalattackdetector/tests/test_unitary.py index 9c05d05f679ea73d40967c3e881aeceb1b3769bf..5aadbe9b177f68dcceca32fe3c4ac57c4fe00163 100644 --- a/src/opticalattackdetector/tests/test_unitary.py +++ b/src/opticalattackdetector/tests/test_unitary.py @@ -14,10 +14,13 @@ import logging, pytest from unittest.mock import patch -from opticalattackdetector.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD -from opticalattackdetector.client.OpticalAttackDetectorClient import OpticalAttackDetectorClient + +from common.Constants import DEFAULT_GRPC_BIND_ADDRESS, DEFAULT_GRPC_MAX_WORKERS, DEFAULT_GRPC_GRACE_PERIOD from common.proto.context_pb2 import ContextIdList, ContextId, Empty, Service, ContextId, ServiceList from common.proto.monitoring_pb2 import Kpi, KpiList + +from opticalattackdetector.Config import GRPC_SERVICE_PORT +from opticalattackdetector.client.OpticalAttackDetectorClient import OpticalAttackDetectorClient from opticalattackdetector.service.OpticalAttackDetectorService import OpticalAttackDetectorService from .example_objects import CONTEXT_ID, CONTEXT_ID_2, SERVICE_DEV1_DEV2 @@ -29,7 +32,7 @@ LOGGER.setLevel(logging.DEBUG) @pytest.fixture(scope='session') def optical_attack_detector_service(): _service = OpticalAttackDetectorService( - port=port, max_workers=GRPC_MAX_WORKERS, grace_period=GRPC_GRACE_PERIOD) + port=port, max_workers=DEFAULT_GRPC_MAX_WORKERS, grace_period=DEFAULT_GRPC_GRACE_PERIOD) # mocker_context_client = mock.patch('opticalattackdetector.service.OpticalAttackDetectorServiceServicerImpl.context_client') # mocker_context_client.start() diff --git a/src/opticalattackmitigator/tests/opticalattackmitigator_report.xml b/src/opticalattackmitigator/tests/opticalattackmitigator_report.xml deleted file mode 100644 index 7d8ea59784f09eb891e9a92c6237eb9d6459dd71..0000000000000000000000000000000000000000 --- a/src/opticalattackmitigator/tests/opticalattackmitigator_report.xml +++ /dev/null @@ -1 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="1" time="0.295" timestamp="2022-07-28T16:09:47.894890" hostname="35c685c0deaa"><testcase classname="opticalattackmitigator.tests.test_unitary" name="test_call_service" time="0.012" /></testsuite></testsuites> \ No newline at end of file diff --git a/tutorial/3-2-develop-cth.md b/tutorial/3-2-develop-cth.md index 6b962be099c04d62563cd66a5fc2abe94eb626ac..a3897ed1304d790c921213ef3d7048d6de8fbdd0 100644 --- a/tutorial/3-2-develop-cth.md +++ b/tutorial/3-2-develop-cth.md @@ -1,17 +1,6 @@ # 3.2. Development Commands, Tricks, and Hints (WORK IN PROGRESS) -## Building, running, testing and reporting code coverage locally -The project runs a CI/CD loops that ensures that all tests are run whenever new code is committed to our reporitory. -However, committing and waiting for the pipeline to run can take substantial time. -For this reason, we prepared a script that runs in your local machine, builds the container image and executes the tests within the image. - -To use the script receives one argument that is the name of the component you want to run. -For instance, if you want to build and run the tests of the `device` component, you can run: - -```shell -scripts/build_run_report_tests.sh device -``` ## Items to be addressed: - pytest flags: --log-level=INFO --verbose -o log_cli=true -o log_file=my_log_file.log -o log_file_level=DEBUG