Updated L3 components + scalability

  • Migrated L3 Cybersecurity to TFS Release 2 architecture
  • Implemented creation of ACL rules when an attack is detected
  • Improved logic to load AI/ML model
  • Updated AI/ML model to detect cybersecurity attacks
  • Added new RPC method to inform the name of the features needs by the model
  • Added new RPC methods to allow exporting ACL rules created by AM to other components.
  • Added logic to measure performance of the L3 Cybersecurity framework
  • Implemented horizontal scalability in L3 Centralized Attack Detector and Attack Mitigator
  • Added Cybersecurity Grafana dashboard
  • Added Service Monitors to export metrics to Prometheus
  • Added multiple helper scripts
  • Multiple bug fixes and code cleanup
Edited by Lluis Gifre Renom

Merge request reports

Loading
+59 −0
Changes for manifests/l3_attackmitigatorservice.yaml: 59 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ spec:
        imagePullPolicy: Always
        ports:
        - containerPort: 10002
        - containerPort: 9192
        env:
        - name: LOG_LEVEL
          value: "DEBUG"
@@ -53,11 +54,69 @@ apiVersion: v1
kind: Service
metadata:
  name: l3-attackmitigatorservice
  labels:
    app: l3-attackmitigatorservice
spec:
  type: ClusterIP
  selector:
    app: l3-attackmitigatorservice
  ports:
  - name: metrics
    protocol: TCP
    port: 9192
    targetPort: 9192
  - name: grpc
    port: 10002
    targetPort: 10002

---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: l3-attackmitigatorservice-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: l3-attackmitigatorservice
  minReplicas: 1
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 80
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 120

---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: tfs-l3-attackmitigatorservice-metric
  labels:
    app: l3-attackmitigatorservice
    #release: prometheus
    #release: prom  # name of the release 
    # ( VERY IMPORTANT: You need to know the correct release name by viewing 
    #   the servicemonitor of Prometheus itself: Without the correct name, 
    #   Prometheus cannot identify the metrics of the Flask app as the target.)
spec:
  selector:
    matchLabels:
      # Target app service
      #namespace: tfs
      app: l3-attackmitigatorservice # same as above
      #release: prometheus # same as above
  endpoints:
  - port: metrics # named port in target app
    scheme: http
    path: /metrics # path to scrape
    interval: 5s # scrape interval
  namespaceSelector:
    any: false
    matchNames:
    - tfs # namespace where the app is running
+58 −0
Changes for manifests/l3_centralizedattackdetectorservice.yaml: 58 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ spec:
        imagePullPolicy: Always
        ports:
        - containerPort: 10001
        - containerPort: 9192
        env:
        - name: LOG_LEVEL
          value: "DEBUG"
@@ -53,11 +54,68 @@ apiVersion: v1
kind: Service
metadata:
  name: l3-centralizedattackdetectorservice
  labels:
    app: l3-centralizedattackdetectorservice
spec:
  type: ClusterIP
  selector:
    app: l3-centralizedattackdetectorservice
  ports:
  - name: metrics
    protocol: TCP
    port: 9192
    targetPort: 9192
  - name: grpc
    port: 10001
    targetPort: 10001

---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: l3-centralizedattackdetectorservice-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: l3-centralizedattackdetectorservice
  minReplicas: 1
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 80
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 120
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: tfs-l3-centralizedattackdetectorservice-metric
  labels:
    app: l3-centralizedattackdetectorservice
    #release: prometheus
    #release: prom  # name of the release 
    # ( VERY IMPORTANT: You need to know the correct release name by viewing 
    #   the servicemonitor of Prometheus itself: Without the correct name, 
    #   Prometheus cannot identify the metrics of the Flask app as the target.)
spec:
  selector:
    matchLabels:
      # Target app service
      #namespace: tfs
      app: l3-centralizedattackdetectorservice # same as above
      #release: prometheus # same as above
  endpoints:
  - port: metrics # named port in target app
    scheme: http
    path: /metrics # path to scrape
    interval: 5s # scrape interval
  namespaceSelector:
    any: false
    matchNames:
    - tfs # namespace where the app is running
+21 −11
Changes for proto/l3_attackmitigator.proto: 21 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -17,10 +17,12 @@ syntax = "proto3";
import "context.proto";

service L3Attackmitigator{
  // Sends a greeting
  rpc SendOutput (L3AttackmitigatorOutput) returns (context.Empty) {}
  // Sends another greeting
  // Perform Mitigation
  rpc PerformMitigation (L3AttackmitigatorOutput) returns (context.Empty) {}
  // Get Mitigation
  rpc GetMitigation (context.Empty) returns (context.Empty) {}
  // Get Configured ACL Rules
  rpc GetConfiguredACLRules (context.Empty) returns (ACLRules) {}
}


@@ -28,12 +30,20 @@ message L3AttackmitigatorOutput {
	float confidence = 1;
	string timestamp = 2;
	string ip_o = 3;
	string tag_name = 4;
	int32 tag = 5;
	string flow_id = 6;
	string protocol = 7;
	string port_d = 8;
	string ml_id = 9;
	float time_start = 10;
	float time_end = 11;
	string ip_d = 4;	
	string tag_name = 5;
	int32 tag = 6;
	string flow_id = 7;
	string protocol = 8;
	string port_o = 9;
	string port_d = 10;
	string ml_id = 11;
	context.ServiceId service_id = 12;
	context.EndPointId endpoint_id = 13; 
	float time_start = 14;
	float time_end = 15;
}

message ACLRules {
	repeated context.ConfigRule acl_rules = 1;
}
+37 −47
Changes for proto/l3_centralizedattackdetector.proto: 37 added lines, 47 removed lines.
Original line number Diff line number Diff line
@@ -14,65 +14,55 @@

syntax = "proto3";

import "context.proto";

service L3Centralizedattackdetector {
  // Sends a greeting
  rpc SendInput (L3CentralizedattackdetectorMetrics) returns (Empty) {}
  // Sends another greeting
  rpc GetOutput (Empty) returns (L3CentralizedattackdetectorModelOutput) {}
  // Analyze single input to the ML model in the CAD component
  rpc AnalyzeConnectionStatistics (L3CentralizedattackdetectorMetrics) returns (Empty) {}

  // Analyze a batch of inputs to the ML model in the CAD component
  rpc AnalyzeBatchConnectionStatistics (L3CentralizedattackdetectorBatchInput) returns (Empty) {}

  // Get the list of features used by the ML model in the CAD component
  rpc GetFeaturesIds (Empty) returns (AutoFeatures) {}
}

message Feature {
	float feature = 1;
}

message L3CentralizedattackdetectorMetrics {
	/*
	Model input sent to the Inferencer by the client
	There are currently 9 values and 
	*/
	// Input sent by the DAD compoenent to the ML model integrated in the CAD component.

    // Machine learning
	float n_packets_server_seconds = 1;
	float n_packets_client_seconds = 2;
	float n_bits_server_seconds = 3;
	float n_bits_client_seconds = 4;
	float n_bits_server_n_packets_server = 5;
	float n_bits_client_n_packets_client = 6;
	float n_packets_server_n_packets_client = 7;
	float n_bits_server_n_bits_client = 8;
	// Machine learning model features
	repeated Feature features = 1;	
	ConnectionMetadata connection_metadata = 2;

	// Conection identifier
	string ip_o = 9;
	string port_o = 10;
	string ip_d = 11;
	string port_d = 12;
	string flow_id = 13;
	string protocol = 14;
	float time_start = 15;
	float time_end = 16;
}

message Empty {
	string message = 1;
message ConnectionMetadata {
	string ip_o = 1;
	string port_o = 2;
	string ip_d = 3;
	string port_d = 4;
	string flow_id = 5;
	context.ServiceId service_id = 6;
	context.EndPointId endpoint_id = 7;
	string protocol = 8;
	float time_start = 9;
	float time_end = 10;
}

message L3CentralizedattackdetectorModelOutput {
	float confidence = 1;
	string timestamp = 2;
	string ip_o = 3;	
	string tag_name = 4;
	int32 tag = 5;
	string flow_id = 6;
	string protocol = 7;
	string port_d = 8;
	string ml_id = 9;
	float time_start = 10;
	float time_end = 11;
// Collection of values representing ML features
message AutoFeatures {
	repeated float auto_features = 1;
}

// Collections or streams?
/*
message InputCollection {
	repeated model_input = 1;
// Collection (batch) of model inputs that will be sent to the model
message L3CentralizedattackdetectorBatchInput {
	repeated L3CentralizedattackdetectorMetrics metrics = 1;
}

message OutputCollection {
	repeated model_output = 1;
message Empty {
	string message = 1;
}
*/
+27 −0
Changes for scripts/show_logs_l3-attack-mitigator.sh: 27 added lines, 0 removed lines.
Original line number Diff line number Diff line
#!/bin/bash
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

########################################################################################################################
# Define your deployment settings here
########################################################################################################################

# If not already set, set the name of the Kubernetes namespace to deploy to.
export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}

########################################################################################################################
# Automated steps start here
########################################################################################################################

kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/l3-attackmitigatorservice -c server
Loading
Loading