Commit 97c50e87 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'develop' of https://labs.etsi.org/rep/tfs/controller into feat/l3-components

parents 2de16332 49d35185

src/opticalattackmanager/service/__main__.py

+4 −19
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ SERVICE_LIST_MODE = int( 
    get_setting("OPTICALATTACKMANAGER_SERVICE_LIST_MODE", default=1)

)

SERVICE_LIST_KEY = get_setting(

    "OPTICALATTACKMANAGER_SERVICE_LIST_KEY", default="opt-sec:active-services"

    "OPTICALATTACKMANAGER_SERVICE_LIST_KEY", default="opt-sec-active-services"

)

MIN_NUMBER_WORKERS = int(

    get_setting("OPTICALATTACKMANAGERSERVICE_LOOP_MIN_WORKERS", default=2)
@@ -295,6 +295,7 @@ async def monitor_services(terminate, service_list=None, cache=None): 


        current_list = []

        if SERVICE_LIST_MODE == LIST_REDIS_MODE:

            LOGGER.debug(f"Services at the Redis DB: {cache.llen(SERVICE_LIST_KEY)}")

            current_list.extend(

                [

                    pickle.loads(service)
@@ -332,22 +333,6 @@ async def monitor_services(terminate, service_list=None, cache=None): 
            )

        )



        # start standard implementation

        # tasks = []

        # for service in current_list:

        #     aw = detect_attack(

        #         host,

        #         port,

        #         service["context"],

        #         service["service"],

        #         service["kpi"],

        #         # allow at most 90% of the monitoring interval to succeed

        #         monitoring_interval * 0.9,

        #     )

        #     tasks.append(aw)

        # [await aw for aw in tasks]

        # end standard implementation



        # start pool implementation

        if len(current_list) == 0:  # guard clause to re-check if services still there

            LOGGER.info(
@@ -385,7 +370,6 @@ async def monitor_services(terminate, service_list=None, cache=None): 
                (i + 1) * k + min(i + 1, m),  # last index

                host,

                port,

                DROP_COUNTER,

                desired_monitoring_interval * 0.9,

            )

            for i in range(cur_number_workers)
@@ -476,6 +460,7 @@ def main(): 
    if SERVICE_LIST_MODE == LIST_REDIS_MODE:

        cache = redis.Redis(host=redis_host, port=redis_port, password=redis_password)

        cache.ping()

        LOGGER.info(f"Connecting to Redis: host={redis_host}, port={redis_port}, password={redis_password}")



        # clean the existing list that will be populated later on in this function

        cache.delete(SERVICE_LIST_KEY)
@@ -544,7 +529,7 @@ def main(): 
    # asyncio.create_task(monitor_services(service_list))



    # Wait for Ctrl+C or termination signal

    while not terminate.wait(timeout=1):

    while not terminate.wait(timeout=10):

        pass



    LOGGER.info("Terminating...")

src/opticalattackmanager/utils/monitor.py

+3 −6
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 


import asyncio

import logging

import traceback



from grpclib.client import Channel

from prometheus_client import Counter
@@ -31,7 +32,6 @@ async def detect_attack( 
    context_id: str,

    service_id: str,

    kpi_id: str,

    drop_counter: Counter,

    timeout: float = 20.0,

) -> None:

    try:
@@ -49,10 +49,9 @@ async def detect_attack( 
        LOGGER.debug("Monitoring finished for {}/{}".format(service_id, kpi_id))

    except Exception as e:

        LOGGER.warning(

            "Exception while processing service_id {}/{}".format(service_id, kpi_id)

            "Exception while processing service_id {}/{}: {}".format(service_id, kpi_id, e)

        )

        # LOGGER.exception(e)

        drop_counter.inc()

        traceback.print_exc()





def delegate_services(
@@ -61,7 +60,6 @@ def delegate_services( 
    end_index: int,

    host: str,

    port: str,

    drop_counter: Counter,

    monitoring_interval: float,

):

    async def run_internal_loop():
@@ -73,7 +71,6 @@ def delegate_services( 
                service["context"],

                service["service"],

                service["kpi"],

                drop_counter,

                # allow at most 90% of the monitoring interval to succeed

                monitoring_interval * 0.9,

            )

src/tests/ofc22/deploy_specs.sh

+2 −2
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ export CRDB_DATABASE="tfs" 
export CRDB_DEPLOY_MODE="single"



# Disable flag for dropping database, if it exists.

export CRDB_DROP_DATABASE_IF_EXISTS=""

export CRDB_DROP_DATABASE_IF_EXISTS="YES"



# Disable flag for re-deploying CockroachDB from scratch.

export CRDB_REDEPLOY=""
@@ -87,7 +87,7 @@ export QDB_PASSWORD="quest" 
export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis"



# Disable flag for dropping tables if they exist.

export QDB_DROP_TABLES_IF_EXIST=""

export QDB_DROP_TABLES_IF_EXIST="YES"



# Disable flag for re-deploying QuestDB from scratch.

export QDB_REDEPLOY=""

src/tests/scenario3/optical/dashboard.json

+184 −175
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ 
  "liveNow": false,

  "panels": [

    {

      "collapsed": false,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -73,6 +74,7 @@ 
        "y": 0

      },

      "id": 12,

      "panels": [],

      "title": "General status",

      "type": "row"

    },
@@ -216,7 +218,7 @@ 
      "options": {

        "legend": {

          "calcs": [],

          "displayMode": "list",

          "displayMode": "hidden",

          "placement": "bottom"

        },

        "tooltip": {
@@ -230,7 +232,11 @@ 
            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "expr": "tfs_opticalattackmanager_dropped_assessments_created",

          "editorMode": "code",

          "exemplar": false,

          "expr": "rate(tfs_opticalattackmanager_dropped_assessments_created[1m])",

          "instant": false,

          "range": true,

          "refId": "A"

        }

      ],
@@ -238,6 +244,7 @@ 
      "type": "timeseries"

    },

    {

      "collapsed": false,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -245,6 +252,7 @@ 
        "y": 9

      },

      "id": 10,

      "panels": [],

      "title": "Pipeline",

      "type": "row"

    },
@@ -253,6 +261,7 @@ 
        "type": "prometheus",

        "uid": "${DS_PROMETHEUS}"

      },

      "description": "",

      "fieldConfig": {

        "defaults": {

          "color": {
@@ -309,11 +318,11 @@ 
        "x": 0,

        "y": 10

      },

      "id": 16,

      "id": 14,

      "options": {

        "legend": {

          "calcs": [],

          "displayMode": "list",

          "displayMode": "hidden",

          "placement": "bottom"

        },

        "tooltip": {
@@ -328,25 +337,13 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackmanager_loop_seconds_bucket[$__rate_interval])) by (le))",

          "legendFormat": "Measured",

          "expr": "tfs_opticalattackmanager_active_services",

          "legendFormat": "Active services",

          "range": true,

          "refId": "A"

        },

        {

          "datasource": {

            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_desired_monitoring_interval",

          "hide": false,

          "legendFormat": "Desired",

          "range": true,

          "refId": "B"

        }

      ],

      "title": "Loop time",

      "title": "Number of active optical services",

      "type": "timeseries"

    },

    {
@@ -354,7 +351,6 @@ 
        "type": "prometheus",

        "uid": "${DS_PROMETHEUS}"

      },

      "description": "",

      "fieldConfig": {

        "defaults": {

          "color": {
@@ -411,7 +407,7 @@ 
        "x": 12,

        "y": 10

      },

      "id": 14,

      "id": 16,

      "options": {

        "legend": {

          "calcs": [],
@@ -430,13 +426,25 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_active_services",

          "legendFormat": "Active services",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackmanager_loop_seconds_bucket[$__rate_interval])) by (le))",

          "legendFormat": "Measured",

          "range": true,

          "refId": "A"

        },

        {

          "datasource": {

            "type": "prometheus",

            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "tfs_opticalattackmanager_desired_monitoring_interval",

          "hide": false,

          "legendFormat": "Desired",

          "range": true,

          "refId": "B"

        }

      ],

      "title": "Number of active optical services",

      "title": "Loop time",

      "type": "timeseries"

    },

    {
@@ -543,7 +551,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_inference_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"uldetection\"}[$__rate_interval])) by (le))",

          "hide": false,

          "legendFormat": "UL Inference",

          "range": true,
@@ -555,7 +563,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_cache_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"cachefetch\"}[$__rate_interval])) by (le))",

          "hide": false,

          "interval": "",

          "legendFormat": "Cache",
@@ -568,7 +576,7 @@ 
            "uid": "${DS_PROMETHEUS}"

          },

          "editorMode": "code",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_mitigation_response_time_bucket[$__rate_interval])) by (le))",

          "expr": "histogram_quantile(0.95, sum(rate(tfs_opticalattackdetector_execution_details_histogram_duration_bucket{step=\"mitigation\"}[$__rate_interval])) by (le))",

          "hide": false,

          "interval": "",

          "legendFormat": "Mitigator",
@@ -1353,7 +1361,7 @@ 
      "type": "timeseries"

    },

    {

      "collapsed": false,

      "collapsed": true,

      "gridPos": {

        "h": 1,

        "w": 24,
@@ -1361,10 +1369,7 @@ 
        "y": 44

      },

      "id": 2,

      "panels": [],

      "title": "General status",

      "type": "row"

    },

      "panels": [

        {

          "datasource": {

            "type": "prometheus",
@@ -1517,7 +1522,11 @@ 
          "type": "timeseries"

        }

      ],

  "refresh": "5s",

      "title": "General status",

      "type": "row"

    }

  ],

  "refresh": "",

  "schemaVersion": 36,

  "style": "dark",

  "tags": [],
@@ -1525,13 +1534,13 @@ 
    "list": []

  },

  "time": {

    "from": "now-5m",

    "from": "now-30m",

    "to": "now"

  },

  "timepicker": {},

  "timezone": "",

  "title": "TFS / Optical cybersecurity",

  "uid": "-Q-B-AsVk",

  "version": 1,

  "version": 5,

  "weekStart": ""

}
 
 No newline at end of file

src/tests/scenario3/optical/deploy_specs.sh

+2 −0
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ export TFS_COMPONENTS="context device automation monitoring pathcomp service sli 


# addition for the optical cybersecurity component

export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager"



export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicemonitors.yaml"

export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml"