Commit f528aebf authored by Pablo Armingol's avatar Pablo Armingol
Browse files

Merge branch 'feat/device-perf-eval' of... 

Merge branch 'feat/device-perf-eval' of https://labs.etsi.org/rep/tfs/controller into perf/tid-openconfig
parents ddc81535 32e1a14f

src/common/method_wrappers/Decorator.py

+6 −1
Original line number Diff line number Diff line
@@ -45,10 +45,14 @@ class MetricsPool: 
    lock = threading.Lock()

    metrics : Dict[str, MetricWrapperBase] = dict()



    def __init__(self, component : str, sub_module : str, labels : Dict[str, str] = {}) -> None:

    def __init__(

        self, component : str, sub_module : str, labels : Dict[str, str] = {},

        default_metric_params : Dict[MetricTypeEnum, Dict] = dict()

    ) -> None:

        self._component = component

        self._sub_module = sub_module

        self._labels = labels

        self._default_metric_params = default_metric_params



    def get_or_create(self, method : str, metric_type : MetricTypeEnum, **metric_params) -> MetricWrapperBase:

        metric_name = str(metric_type.value).format(
@@ -57,6 +61,7 @@ class MetricsPool: 
            if metric_name not in MetricsPool.metrics:

                metric_tuple : Tuple[MetricWrapperBase, Dict] = METRIC_TO_CLASS_PARAMS.get(metric_type)

                metric_class, default_metric_params = metric_tuple

                if len(metric_params) == 0: metric_params = self._default_metric_params.get(metric_type, {})

                if len(metric_params) == 0: metric_params = default_metric_params

                labels = sorted(self._labels.keys())

                MetricsPool.metrics[metric_name] = metric_class(metric_name.lower(), '', labels, **metric_params)

src/common/method_wrappers/tests/README.md

+28 −16
Original line number Diff line number Diff line
@@ -5,7 +5,16 @@ 
- enable prometheus addon:

```

tfs@tfs-vm:~/tfs-ctrl$ microk8s.enable prometheus

tfs@tfs-vm:~/tfs-ctrl$ microk8s.status --wait-ready

```



- wait till prometheus becomes enabled (when enabled, press Ctrl+C):

```

tfs@tfs-vm:~/tfs-ctrl$ watch -n 1 microk8s.status --wait-ready

```



- wait till all pods in the monitoring namespace have STATE=Running and READY=X/X (when done, press Ctrl+C):

```

tfs@tfs-vm:~/tfs-ctrl$ watch -n 1 kubectl get pods --all-namespaces

```



- deploy as:
@@ -15,23 +24,26 @@ tfs@tfs-vm:~/tfs-ctrl$ ./deploy.sh 
```



- expose prometheus and grafana

  - (required) terminal 1 (grafana UI): `kubectl port-forward -n monitoring service/grafana --address 0.0.0.0 3001:3000`

  - (optional) terminal 2 (prometheus UI): `kubectl port-forward -n monitoring service/prometheus-k8s --address 0.0.0.0 9090:9090`

  - (optional) terminal 3 (alertmanager UI): `kubectl port-forward -n monitoring service/alertmanager-main --address 0.0.0.0 9093:9093`



  - terminal 1 (prometheus UI): `kubectl port-forward -n monitoring service/prometheus-k8s --address 0.0.0.0 9090:9090`

  - terminal 2 (grafana UI): `kubectl port-forward -n monitoring service/grafana --address 0.0.0.0 3001:3000`

  - terminal 3 (alertmanager UI): `kubectl port-forward -n monitoring service/alertmanager-main --address 0.0.0.0 9093:9093`



- if using remote server/VM for running MicroK8s and VSCode, forward ports 9090, 3000, 9093

- if using remote server/VM for running MicroK8s and VSCode, forward ports 3001, 9090, 9093



terminal 4 (tun test_set):

- (only used for internal framework debugging) run manual tests over the performance evaluation framework

  - terminal 4:

  ```

  export PYTHONPATH=/home/tfs/tfs-ctrl/src

python -m common.method_wrappers.tests.test_set

  python -m common.method_wrappers.tests

  ```



- log into grafana:

  - 127.0.0.1:3000

  - admin/admin

  - upload dashboard_prometheus_histogram.json

  - browse: http://127.0.0.1:3000

  - user/pass: admin/admin

  - upload dashboards through "left menu > Dashboards > Manage > Import"

    - upload grafana_prometheus_component_rpc.json

    - upload grafana_prometheus_device_driver.json

    - upload grafana_prometheus_service_handler.json

  - watch in real time the dashboard



- upload topology through WebUI and navigate

src/device/service/drivers/emulated/EmulatedDriver.py

+17 −1
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ from apscheduler.executors.pool import ThreadPoolExecutor 
from apscheduler.job import Job

from apscheduler.jobstores.memory import MemoryJobStore

from apscheduler.schedulers.background import BackgroundScheduler

from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method

from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF

from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk_type

from device.service.database.KpiSampleType import ORM_KpiSampleTypeEnum, grpc_to_enum__kpi_sample_type

from device.service.driver_api._Driver import (
@@ -123,7 +123,23 @@ def do_sampling( 
    value     = abs(0.95 * waveform + 0.05 * noise)

    out_samples.put_nowait((timestamp, resource_key, value))



HISTOGRAM_BUCKETS = (

    # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF

    0.0001, 0.00025, 0.00050, 0.00075,

    0.0010, 0.0025, 0.0050, 0.0075,

    0.0100, 0.0250, 0.0500, 0.0750,

    0.1000, 0.2500, 0.5000, 0.7500,

    1.0000, 2.5000, 5.0000, 7.5000,

    10.0, 25.0, 50.0, 75.0,

    100.0, INF

)

METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'emulated'})

METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('GetConfig',        MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SetConfig',        MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('DeleteConfig',     MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SubscribeState',   MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)



class EmulatedDriver(_Driver):

    def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called

src/device/service/drivers/openconfig/OpenConfigDriver.py

+17 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ from apscheduler.job import Job 
from apscheduler.jobstores.memory import MemoryJobStore

from apscheduler.schedulers.background import BackgroundScheduler

from ncclient.manager import Manager, connect_ssh

from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method

from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF

from common.tools.client.RetryDecorator import delay_exponential

from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float

from device.service.driver_api.Exceptions import UnsupportedResourceKeyException
@@ -223,7 +223,23 @@ def edit_config( 
            results[i] = e # if validation fails, store the exception

    return results



HISTOGRAM_BUCKETS = (

    # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF

    0.0001, 0.00025, 0.00050, 0.00075,

    0.0010, 0.0025, 0.0050, 0.0075,

    0.0100, 0.0250, 0.0500, 0.0750,

    0.1000, 0.2500, 0.5000, 0.7500,

    1.0000, 2.5000, 5.0000, 7.5000,

    10.0, 25.0, 50.0, 75.0,

    100.0, INF

)

METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'openconfig'})

METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('GetConfig',        MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SetConfig',        MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('DeleteConfig',     MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SubscribeState',   MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)



class OpenConfigDriver(_Driver):

    def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called

src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py

+16 −1
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 


import anytree, json, logging

from typing import Any, List, Optional, Tuple, Union

from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method

from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF

from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service

from common.tools.object_factory.Device import json_device_id

from common.type_checkers.Checkers import chk_length, chk_type
@@ -25,7 +25,22 @@ from .ConfigRules import setup_config_rules, teardown_config_rules 


LOGGER = logging.getLogger(__name__)



HISTOGRAM_BUCKETS = (

    # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF

    0.0010, 0.0025, 0.0050, 0.0075,

    0.0100, 0.0250, 0.0500, 0.0750,

    0.1000, 0.2500, 0.5000, 0.7500,

    1.0000, 2.5000, 5.0000, 7.5000,

    10.0000, 25.000, 50.0000, 75.000,

    100.0, INF

)

METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l2nm_emulated'})

METRICS_POOL.get_or_create('SetEndpoint',      MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('DeleteEndpoint',   MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SetConstraint',    MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('SetConfig',        MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)

METRICS_POOL.get_or_create('DeleteConfig',     MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS)



class L2NMEmulatedServiceHandler(_ServiceHandler):

    def __init__(   # pylint: disable=super-init-not-called