Commit 49d35185 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'fix/opt-cybersecurity' into 'develop' 

Improvements to the optical cybersecurity component addressing previous issues

See merge request !127
parents c0b0fb10 55e3b2b1

src/opticalattackmanager/service/__main__.py

+4 −19
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ SERVICE_LIST_MODE = int( 
    get_setting("OPTICALATTACKMANAGER_SERVICE_LIST_MODE", default=1)

)

SERVICE_LIST_KEY = get_setting(

    "OPTICALATTACKMANAGER_SERVICE_LIST_KEY", default="opt-sec:active-services"

    "OPTICALATTACKMANAGER_SERVICE_LIST_KEY", default="opt-sec-active-services"

)

MIN_NUMBER_WORKERS = int(

    get_setting("OPTICALATTACKMANAGERSERVICE_LOOP_MIN_WORKERS", default=2)
@@ -295,6 +295,7 @@ async def monitor_services(terminate, service_list=None, cache=None): 


        current_list = []

        if SERVICE_LIST_MODE == LIST_REDIS_MODE:

            LOGGER.debug(f"Services at the Redis DB: {cache.llen(SERVICE_LIST_KEY)}")

            current_list.extend(

                [

                    pickle.loads(service)
@@ -332,22 +333,6 @@ async def monitor_services(terminate, service_list=None, cache=None): 
            )

        )



        # start standard implementation

        # tasks = []

        # for service in current_list:

        #     aw = detect_attack(

        #         host,

        #         port,

        #         service["context"],

        #         service["service"],

        #         service["kpi"],

        #         # allow at most 90% of the monitoring interval to succeed

        #         monitoring_interval * 0.9,

        #     )

        #     tasks.append(aw)

        # [await aw for aw in tasks]

        # end standard implementation



        # start pool implementation

        if len(current_list) == 0:  # guard clause to re-check if services still there

            LOGGER.info(
@@ -385,7 +370,6 @@ async def monitor_services(terminate, service_list=None, cache=None): 
                (i + 1) * k + min(i + 1, m),  # last index

                host,

                port,

                DROP_COUNTER,

                desired_monitoring_interval * 0.9,

            )

            for i in range(cur_number_workers)
@@ -476,6 +460,7 @@ def main(): 
    if SERVICE_LIST_MODE == LIST_REDIS_MODE:

        cache = redis.Redis(host=redis_host, port=redis_port, password=redis_password)

        cache.ping()

        LOGGER.info(f"Connecting to Redis: host={redis_host}, port={redis_port}, password={redis_password}")



        # clean the existing list that will be populated later on in this function

        cache.delete(SERVICE_LIST_KEY)
@@ -544,7 +529,7 @@ def main(): 
    # asyncio.create_task(monitor_services(service_list))



    # Wait for Ctrl+C or termination signal

    while not terminate.wait(timeout=1):

    while not terminate.wait(timeout=10):

        pass



    LOGGER.info("Terminating...")

src/opticalattackmanager/utils/monitor.py

+3 −6
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 


import asyncio

import logging

import traceback



from grpclib.client import Channel

from prometheus_client import Counter
@@ -31,7 +32,6 @@ async def detect_attack( 
    context_id: str,

    service_id: str,

    kpi_id: str,

    drop_counter: Counter,

    timeout: float = 20.0,

) -> None:

    try:
@@ -49,10 +49,9 @@ async def detect_attack( 
        LOGGER.debug("Monitoring finished for {}/{}".format(service_id, kpi_id))

    except Exception as e:

        LOGGER.warning(

            "Exception while processing service_id {}/{}".format(service_id, kpi_id)

            "Exception while processing service_id {}/{}: {}".format(service_id, kpi_id, e)

        )

        # LOGGER.exception(e)

        drop_counter.inc()

        traceback.print_exc()





def delegate_services(
@@ -61,7 +60,6 @@ def delegate_services( 
    end_index: int,

    host: str,

    port: str,

    drop_counter: Counter,

    monitoring_interval: float,

):

    async def run_internal_loop():
@@ -73,7 +71,6 @@ def delegate_services( 
                service["context"],

                service["service"],

                service["kpi"],

                drop_counter,

                # allow at most 90% of the monitoring interval to succeed

                monitoring_interval * 0.9,

            )

src/tests/scenario3/optical/dashboard.json

+184 −175
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ 
  "liveNow": false,

  "panels": [

    {

      "collapsed": false,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -73,6 +74,7 @@ 
        "y": 0

      },

      "id": 12,

      "panels": [],

      "title": "General status",

      "type": "row"

    },
@@ -216,7 +218,7 @@ 
      "options": {

        "legend": {

          "calcs": [],

          "displayMode": "list",

          "displayMode": "hidden",

          "placement": "bottom"

        },

        "tooltip": {
@@ -230,7 +232,11 @@ 
            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "expr": "tfs_opticalattackmanager_dropped_assessments_created",

          "editorMode": "code",

          "exemplar": false,

          "expr": "rate(tfs_opticalattackmanager_dropped_assessments_created[1m])",

          "instant": false,

          "range": true,

          "refId": "A"

        }

      ],
@@ -238,6 +244,7 @@ 
      "type": "timeseries"

    },

    {

      "collapsed": false,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -245,6 +252,7 @@ 
        "y": 9

      },

      "id": 10,

      "panels": [],

      "title": "Pipeline",

      "type": "row"

    },
@@ -253,6 +261,7 @@ 
        "type": "prometheus",

        "uid": "${DS_PROMETHEUS}"

      },

      "description": "",

      "fieldConfig": {

        "defaults": {

          "color": {
@@ -309,11 +318,11 @@ 
        "x": 0,

        "y": 10

      },

      "id": 16,

      "id": 14,

      "options": {

        "legend": {

          "calcs": [],

          "displayMode": "list",

          "displayMode": "hidden",

          "placement": "bottom"

        },

        "tooltip": {
@@ -328,25 +337,13 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackmanager_loop_seconds_bucket[$__rate_interval])) by (le))",

          "legendFormat": "Measured",

          "expr": "tfs_opticalattackmanager_active_services",

          "legendFormat": "Active services",

          "range": true,

          "refId": "A"

        },

        {

          "datasource": {

            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_desired_monitoring_interval",

          "hide": false,

          "legendFormat": "Desired",

          "range": true,

          "refId": "B"

        }

      ],

      "title": "Loop time",

      "title": "Number of active optical services",

      "type": "timeseries"

    },

    {
@@ -354,7 +351,6 @@ 
        "type": "prometheus",

        "uid": "${DS_PROMETHEUS}"

      },

      "description": "",

      "fieldConfig": {

        "defaults": {

          "color": {
@@ -411,7 +407,7 @@ 
        "x": 12,

        "y": 10

      },

      "id": 14,

      "id": 16,

      "options": {

        "legend": {

          "calcs": [],
@@ -430,13 +426,25 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_active_services",

          "legendFormat": "Active services",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackmanager_loop_seconds_bucket[$__rate_interval])) by (le))",

          "legendFormat": "Measured",

          "range": true,

          "refId": "A"

        },

        {

          "datasource": {

            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_desired_monitoring_interval",

          "hide": false,

          "legendFormat": "Desired",

          "range": true,

          "refId": "B"

        }

      ],

      "title": "Number of active optical services",

      "title": "Loop time",

      "type": "timeseries"

    },

    {
@@ -543,7 +551,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_inference_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"uldetection\"}[$__rate_interval])) by (le))",

          "hide": false,

          "legendFormat": "UL Inference",

          "range": true,
@@ -555,7 +563,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_cache_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"cachefetch\"}[$__rate_interval])) by (le))",

          "hide": false,

          "interval": "",

          "legendFormat": "Cache",
@@ -568,7 +576,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_mitigation_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"mitigation\"}[$__rate_interval])) by (le))",

          "hide": false,

          "interval": "",

          "legendFormat": "Mitigator",
@@ -1353,7 +1361,7 @@ 
      "type": "timeseries"

    },

    {

      "collapsed": false,

      "collapsed": true,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -1361,10 +1369,7 @@ 
        "y": 44

      },

      "id": 2,

      "panels": [],

      "title": "General status",

      "type": "row"

    },

      "panels": [

        {

          "datasource": {

            "type": "prometheus",
@@ -1517,7 +1522,11 @@ 
          "type": "timeseries"

        }

      ],

  "refresh": "5s",

      "title": "General status",

      "type": "row"

    }

  ],

  "refresh": "",

  "schemaVersion": 36,

  "style": "dark",

  "tags": [],
@@ -1525,13 +1534,13 @@ 
    "list": []

  },

  "time": {

    "from": "now-5m",

    "from": "now-30m",

    "to": "now"

  },

  "timepicker": {},

  "timezone": "",

  "title": "TFS / Optical cybersecurity",

  "uid": "-Q-B-AsVk",

  "version": 1,

  "version": 5,

  "weekStart": ""

}
 
 No newline at end of file

src/tests/scenario3/optical/deploy_specs.sh

+2 −0
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ export TFS_COMPONENTS="context device automation monitoring pathcomp service sli 


# addition for the optical cybersecurity component

export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager"



export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicemonitors.yaml"

export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml"

src/tests/scenario3/optical/jocn/README.md

+3 −1
Original line number Diff line number Diff line
@@ -2,4 +2,6 @@ 


__Authors__: [Carlos Natalino](https://www.chalmers.se/en/persons/carda/), Lluis Gifre Renom, Francisco-Javier Moreno-Muro, Sergio Gonzalez Diaz, Ricard Vilalta, Raul Muñoz, Paolo Monti, and Marija Furdek



Experiments from the JOCN paper.
 
 No newline at end of file
 
The paper has been published as open access: https://doi.org/10.1364/JOCN.482932



This folder contains the experiments published in the JOCN paper.