Merge branch 'feat/opticalcentralizedattackdetector-service' into 'develop'

Commit for the first code freeze

See merge request teraflow-h2020/controller!46

.gitlab-ci.yml

@@ -16,6 +16,9 @@ include:
   - local: '/src/context/.gitlab-ci.yml'
   - local: '/src/device/.gitlab-ci.yml'
   - local: '/src/service/.gitlab-ci.yml'
+  - local: '/src/dbscanserving/.gitlab-ci.yml'
+  - local: '/src/opticalattackmitigator/.gitlab-ci.yml'
+  - local: '/src/opticalcentralizedattackdetector/.gitlab-ci.yml'
   #- local: '/src/tester_integration/.gitlab-ci.yml'
   #- local: '/src/tester_functional/.gitlab-ci.yml'
   #- local: '/src/automation/.gitlab-ci.yml'

deploy_in_kubernetes.sh

@@ -10,7 +10,7 @@ REGISTRY_IMAGE=""
 #REGISTRY_IMAGE="http://my-container-registry.local/"
 
 # Set the list of components you want to build images for, and deploy.
-COMPONENTS="context device automation policy service compute monitoring centralizedattackdetector"
+COMPONENTS="context device automation policy service compute monitoring centralizedattackdetector dbscanserving opticalattackmitigator opticalcentralizedattackdetector" # TODO: include opticalcentralizedattackdetector
 
 # Set the tag you want to use for your images.
 IMAGE_TAG="tf-dev"

install_development_dependencies.sh

@@ -7,7 +7,7 @@ pip install --upgrade pip setuptools wheel pip-tools pylint pytest pytest-benchm
 echo "" > requirements.in
 
 #TODO: include here your component
-COMPONENTS="compute context device monitoring centralizedattackdetector"
+COMPONENTS="compute context device monitoring centralizedattackdetector opticalcentralizedattackdetector opticalattackmitigator dbscanserving"
 
 # compiling dependencies from all components
 for component in $COMPONENTS

manifests/dbscanservingservice.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dbscanservingservice
+spec:
+  selector:
+    matchLabels:
+      app: dbscanservingservice
+  template:
+    metadata:
+      labels:
+        app: dbscanservingservice
+    spec:
+      terminationGracePeriodSeconds: 5
+      containers:
+      - name: server
+        image: registry.gitlab.com/teraflow-h2020/controller/dbscanserving:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 10006
+        env:
+        - name: LOG_LEVEL
+          value: "DEBUG"
+        readinessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10006"]
+        livenessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10006"]
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: 700m
+            memory: 1024Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dbscanservingservice
+spec:
+  type: ClusterIP
+  selector:
+    app: dbscanservingservice
+  ports:
+  - name: grpc
+    port: 10006
+    targetPort: 10006
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dbscanservingservice-public
+  labels:
+    app: dbscanservingservice
+spec:
+  type: NodePort
+  selector:
+    app: dbscanservingservice
+  ports:
+  - name: http
+    protocol: TCP
+    port: 10006
+    targetPort: 10006
+---

manifests/opticalattackmitigatorservice.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: opticalattackmitigatorservice
+spec:
+  selector:
+    matchLabels:
+      app: opticalattackmitigatorservice
+  template:
+    metadata:
+      labels:
+        app: opticalattackmitigatorservice
+    spec:
+      terminationGracePeriodSeconds: 5
+      containers:
+      - name: server
+        image: registry.gitlab.com/teraflow-h2020/controller/opticalattackmitigator:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 10007
+        env:
+        - name: LOG_LEVEL
+          value: "DEBUG"
+        readinessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10007"]
+        livenessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10007"]
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: 700m
+            memory: 1024Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: opticalattackmitigatorservice
+spec:
+  type: ClusterIP
+  selector:
+    app: opticalattackmitigatorservice
+  ports:
+  - name: grpc
+    port: 10007
+    targetPort: 10007
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: opticalattackmitigatorservice-public
+  labels:
+    app: opticalattackmitigatorservice
+spec:
+  type: NodePort
+  selector:
+    app: opticalattackmitigatorservice
+  ports:
+  - name: http
+    protocol: TCP
+    port: 10007
+    targetPort: 10007
+---

manifests/opticalcentralizedattackdetectorservice.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: opticalcentralizedattackdetectorservice
+spec:
+  selector:
+    matchLabels:
+      app: opticalcentralizedattackdetectorservice
+  template:
+    metadata:
+      labels:
+        app: opticalcentralizedattackdetectorservice
+    spec:
+      terminationGracePeriodSeconds: 5
+      containers:
+      - name: server
+        image: registry.gitlab.com/teraflow-h2020/controller/opticalcentralizedattackdetector:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 10005
+        envFrom:
+          - secretRef:
+              name: monitoring-secrets
+        readinessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10005"]
+        livenessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:10005"]
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: 700m
+            memory: 1024Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: opticalcentralizedattackdetectorservice
+spec:
+  type: ClusterIP
+  selector:
+    app: opticalcentralizedattackdetectorservice
+  ports:
+  - name: grpc
+    port: 10005
+    targetPort: 10005
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: opticalcentralizedattackdetectorservice-public
+  labels:
+    app: opticalcentralizedattackdetectorservice
+spec:
+  type: NodePort
+  selector:
+    app: opticalcentralizedattackdetectorservice
+  ports:
+  - name: grpc
+    protocol: TCP
+    port: 10005
+    targetPort: 10005
+---

proto/dbscanserving.proto

+syntax = "proto3";
+
+package dbscanserving;
+
+enum Metric {
+    EUCLIDEAN = 0;
+}
+
+message Sample {
+    repeated float features = 1;
+}
+
+message DetectionRequest {
+    float eps = 1;
+    int32 min_samples = 2;
+    Metric metric = 3;
+    int32 num_samples = 4;
+    int32 num_features = 5;
+    repeated Sample samples = 6;
+    int32 identifier = 7;
+}
+
+message DetectionResponse {
+    repeated int32 cluster_indices = 1;
+}
+
+service Detector {
+    rpc Detect (DetectionRequest) returns (DetectionResponse);
+}

proto/optical_attack_mitigator.proto

+// protocol buffers documentation: https://developers.google.com/protocol-buffers/docs/proto3
+syntax = "proto3";
+package optical_attack_mitigator;
+
+import "context.proto";
+
+service AttackMitigator {
+  rpc NotifyAttack (AttackDescription) returns (AttackResponse) {}
+}
+
+message AttackDescription {
+  context.Uuid cs_id = 1;
+  int32 attack_id = 2;
+  float confidence = 3;
+  string attack_description = 4;
+}
+
+message AttackResponse {
+  context.Uuid cs_id = 1;
+  int32 attack_id = 2;
+  string attack_description = 3;
+  int32 response_strategy_id = 4;
+  string response_strategy_description = 5;
+}

proto/optical_centralized_attack_detector.proto

+// protocol buffers documentation: https://developers.google.com/protocol-buffers/docs/proto3
+syntax = "proto3";
+package centralized_attack_detector;
+
+import "context.proto";
+import "monitoring.proto";
+
+service OpticalCentralizedAttackDetectorService {
+  rpc NotifyServiceUpdate (context.Service   ) returns (context.Empty) {}
+  
+  // rpc that triggers the attack detection loop
+  rpc DetectAttack        (context.Empty     ) returns (context.Empty) {}
+
+  // rpc called by the distributed component to report KPIs
+  rpc ReportSummarizedKpi (monitoring.KpiList) returns (context.Empty) {}
+
+  rpc ReportKpi           (monitoring.KpiList) returns (context.Empty) {}
+}

run_tests_locally.sh

@@ -2,7 +2,7 @@
 
 PROJECTDIR=`pwd`
 
-cd $(dirname $0)/src
+cd $PROJECTDIR/src
 RCFILE=$PROJECTDIR/coverage/.coveragerc
 COVERAGEFILE=$PROJECTDIR/coverage/.coverage
 
@@ -49,6 +49,15 @@ coverage run --rcfile=$RCFILE --append -m pytest -s --log-level=INFO --verbose \
 coverage run --rcfile=$RCFILE --append -m pytest -s --log-level=INFO --verbose \
     l3_attackmitigator/tests/test_unitary.py
 
+coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+    opticalcentralizedattackdetector/tests/test_unitary.py
+
+coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+    dbscanserving/tests/test_unitary.py
+
+coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+    opticalattackmitigator/tests/test_unitary.py
+
 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
     service/tests/test_unitary.py
 

src/dbscanserving/.gitlab-ci.yml

+# build, tag and push the Docker image to the gitlab registry
+build dbscanserving:
+  variables:
+    IMAGE_NAME: 'dbscanserving' # name of the microservice
+    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
+  stage: build
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/
+    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
+    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
+  # after_script:
+  #   - docker rmi $(docker images --quiet --filter=dangling=true)
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
+    - changes:
+      - src/$IMAGE_NAME/**/*.{py,in,yml}
+      - src/$IMAGE_NAME/Dockerfile
+      - src/$IMAGE_NAME/tests/*.py
+      - src/$IMAGE_NAME/tests/Dockerfile
+      - manifests/$IMAGE_NAME.yaml
+      - .gitlab-ci.yml
+
+# apply unit test to the dbscanserving component
+unit test dbscanserving:
+  variables:
+    IMAGE_NAME: 'dbscanserving' # name of the microservice
+    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
+  stage: unit_test
+  needs:
+    - build dbscanserving
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
+    - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
+  script:
+    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
+    - docker run --name $IMAGE_NAME -d -p 10006:10006 --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+    - sleep 5
+    - docker ps -a
+    - docker exec -i $IMAGE_NAME bash -c "pytest --log-level=DEBUG --verbose $IMAGE_NAME/tests/test_unitary.py"
+  after_script:
+    #- docker rm -f $IMAGE_NAME
+    - docker network rm teraflowbridge
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
+    - changes:
+      - src/$IMAGE_NAME/**/*.{py,in,yml}
+      - src/$IMAGE_NAME/Dockerfile
+      - src/$IMAGE_NAME/tests/*.py
+      - src/$IMAGE_NAME/tests/Dockerfile
+      - manifests/$IMAGE_NAMEservice.yaml
+      - .gitlab-ci.yml
+
+
+# Deployment of the dbscanserving service in Kubernetes Cluster
+deploy dbscanserving:
+  variables:
+    IMAGE_NAME: 'dbscanserving' # name of the microservice
+    IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
+  stage: deploy
+  needs:
+    - unit test dbscanserving
+    # - integ_test execute
+  script:
+    - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml'
+    - kubectl version
+    - kubectl get all
+    - kubectl apply -f "manifests/$IMAGE_NAME.yaml"
+    - kubectl get all
+  # environment:
+  #   name: test
+  #   url: https://example.com
+  #   kubernetes:
+  #     namespace: test
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
+      when: manual    
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
+      when: manual
\ No newline at end of file

src/dbscanserving/Config.py

+import logging
+
+# General settings
+LOG_LEVEL = logging.DEBUG
+
+# gRPC settings
+GRPC_SERVICE_PORT = 10006
+GRPC_MAX_WORKERS  = 10
+GRPC_GRACE_PERIOD = 60
+
+# Prometheus settings
+METRICS_PORT = 9192

src/dbscanserving/Dockerfile

+FROM python:3-slim
+
+# Install dependencies
+RUN apt-get --yes --quiet --quiet update && \
+    apt-get --yes --quiet --quiet install wget g++ && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set Python to show logs as they occur
+ENV PYTHONUNBUFFERED=0
+
+# Download the gRPC health probe
+RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
+
+# Get generic Python packages
+RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools
+
+# Set working directory
+WORKDIR /var/teraflow
+
+# Create module sub-folders
+RUN mkdir -p /var/teraflow/dbscanserving
+
+# Get Python packages per module
+COPY dbscanserving/requirements.in dbscanserving/requirements.in
+RUN pip-compile --output-file=dbscanserving/requirements.txt dbscanserving/requirements.in
+RUN python3 -m pip install -r dbscanserving/requirements.txt
+
+COPY common/. common
+COPY dbscanserving/. dbscanserving
+
+# Start dbscanserving service
+ENTRYPOINT ["python", "-m", "dbscanserving.service"]

src/dbscanserving/client/DbscanServingClient.py

+import grpc, logging
+from common.tools.client.RetryDecorator import retry, delay_exponential
+from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
+from dbscanserving.proto.dbscanserving_pb2_grpc import DetectorStub
+
+LOGGER = logging.getLogger(__name__)
+MAX_RETRIES = 15
+DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
+
+class DbscanServingClient:
+    def __init__(self, address, port):
+        self.endpoint = '{:s}:{:s}'.format(str(address), str(port))
+        LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint)))
+        self.channel = None
+        self.stub = None
+        self.connect()
+        LOGGER.debug('Channel created')
+
+    def connect(self):
+        self.channel = grpc.insecure_channel(self.endpoint)
+        self.stub = DetectorStub(self.channel)
+
+    def close(self):
+        if(self.channel is not None): self.channel.close()
+        self.channel = None
+        self.stub = None
+
+    @retry(exceptions=set(), max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
+    def Detect(self, request : DetectionRequest) -> DetectionResponse:
+        LOGGER.debug('Detect request: {:s}'.format(str(request)))
+        response = self.stub.Detect(request)
+        LOGGER.debug('Detect result: {:s}'.format(str(response)))
+        return response