diff --git a/src/common/method_wrappers/Decorator.py b/src/common/method_wrappers/Decorator.py index 01c256ff6a1815e8a3d027afed897cf2e3878550..7ee2a919e10f25104d0fa77caaf8bafa11c2b30f 100644 --- a/src/common/method_wrappers/Decorator.py +++ b/src/common/method_wrappers/Decorator.py @@ -45,10 +45,14 @@ class MetricsPool: lock = threading.Lock() metrics : Dict[str, MetricWrapperBase] = dict() - def __init__(self, component : str, sub_module : str, labels : Dict[str, str] = {}) -> None: + def __init__( + self, component : str, sub_module : str, labels : Dict[str, str] = {}, + default_metric_params : Dict[MetricTypeEnum, Dict] = dict() + ) -> None: self._component = component self._sub_module = sub_module self._labels = labels + self._default_metric_params = default_metric_params def get_or_create(self, method : str, metric_type : MetricTypeEnum, **metric_params) -> MetricWrapperBase: metric_name = str(metric_type.value).format( @@ -57,6 +61,7 @@ class MetricsPool: if metric_name not in MetricsPool.metrics: metric_tuple : Tuple[MetricWrapperBase, Dict] = METRIC_TO_CLASS_PARAMS.get(metric_type) metric_class, default_metric_params = metric_tuple + if len(metric_params) == 0: metric_params = self._default_metric_params.get(metric_type, {}) if len(metric_params) == 0: metric_params = default_metric_params labels = sorted(self._labels.keys()) MetricsPool.metrics[metric_name] = metric_class(metric_name.lower(), '', labels, **metric_params) diff --git a/src/common/method_wrappers/tests/README.md b/src/common/method_wrappers/tests/README.md index b0db20bd9f33e3e9fd1f1deac55118d9e3e45bde..db9c0687098981d5410326c1330294931f496e3c 100644 --- a/src/common/method_wrappers/tests/README.md +++ b/src/common/method_wrappers/tests/README.md @@ -5,33 +5,45 @@ - enable prometheus addon: ``` tfs@tfs-vm:~/tfs-ctrl$ microk8s.enable prometheus -tfs@tfs-vm:~/tfs-ctrl$ microk8s.status --wait-ready +``` + +- wait till prometheus becomes enabled (when enabled, press Ctrl+C): +``` +tfs@tfs-vm:~/tfs-ctrl$ watch -n 1 microk8s.status --wait-ready +``` + +- wait till all pods in the monitoring namespace have STATE=Running and READY=X/X (when done, press Ctrl+C): +``` +tfs@tfs-vm:~/tfs-ctrl$ watch -n 1 kubectl get pods --all-namespaces ``` - deploy as: ``` -tfs@tfs-vm:~/tfs-ctrl$ source src/common/method_wrappers/tests/deploy_specs.sh -tfs@tfs-vm:~/tfs-ctrl$ ./deploy.sh +tfs@tfs-vm:~/tfs-ctrl$ source src/common/method_wrappers/tests/deploy_specs.sh +tfs@tfs-vm:~/tfs-ctrl$ ./deploy.sh ``` - expose prometheus and grafana + - (required) terminal 1 (grafana UI): `kubectl port-forward -n monitoring service/grafana --address 0.0.0.0 3001:3000` + - (optional) terminal 2 (prometheus UI): `kubectl port-forward -n monitoring service/prometheus-k8s --address 0.0.0.0 9090:9090` + - (optional) terminal 3 (alertmanager UI): `kubectl port-forward -n monitoring service/alertmanager-main --address 0.0.0.0 9093:9093` - - terminal 1 (prometheus UI): `kubectl port-forward -n monitoring service/prometheus-k8s --address 0.0.0.0 9090:9090` - - terminal 2 (grafana UI): `kubectl port-forward -n monitoring service/grafana --address 0.0.0.0 3001:3000` - - terminal 3 (alertmanager UI): `kubectl port-forward -n monitoring service/alertmanager-main --address 0.0.0.0 9093:9093` - -- if using remote server/VM for running MicroK8s and VSCode, forward ports 9090, 3000, 9093 +- if using remote server/VM for running MicroK8s and VSCode, forward ports 3001, 9090, 9093 -terminal 4 (tun test_set): -``` -export PYTHONPATH=/home/tfs/tfs-ctrl/src -python -m common.method_wrappers.tests.test_set -``` +- (only used for internal framework debugging) run manual tests over the performance evaluation framework + - terminal 4: + ``` + export PYTHONPATH=/home/tfs/tfs-ctrl/src + python -m common.method_wrappers.tests + ``` - log into grafana: - - 127.0.0.1:3000 - - admin/admin - - upload dashboard_prometheus_histogram.json + - browse: http://127.0.0.1:3000 + - user/pass: admin/admin + - upload dashboards through "left menu > Dashboards > Manage > Import" + - upload grafana_prometheus_component_rpc.json + - upload grafana_prometheus_device_driver.json + - upload grafana_prometheus_service_handler.json - watch in real time the dashboard - upload topology through WebUI and navigate diff --git a/src/device/service/drivers/emulated/EmulatedDriver.py b/src/device/service/drivers/emulated/EmulatedDriver.py index e980bbc4347f07ea524c93f32b1796e6174ec238..6029ff6604b2525b4509a24a2ec0d6f7c38513d0 100644 --- a/src/device/service/drivers/emulated/EmulatedDriver.py +++ b/src/device/service/drivers/emulated/EmulatedDriver.py @@ -19,7 +19,7 @@ from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.job import Job from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.background import BackgroundScheduler -from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk_type from device.service.database.KpiSampleType import ORM_KpiSampleTypeEnum, grpc_to_enum__kpi_sample_type from device.service.driver_api._Driver import ( @@ -123,7 +123,23 @@ def do_sampling( value = abs(0.95 * waveform + 0.05 * noise) out_samples.put_nowait((timestamp, resource_key, value)) +HISTOGRAM_BUCKETS = ( + # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF + 0.0001, 0.00025, 0.00050, 0.00075, + 0.0010, 0.0025, 0.0050, 0.0075, + 0.0100, 0.0250, 0.0500, 0.0750, + 0.1000, 0.2500, 0.5000, 0.7500, + 1.0000, 2.5000, 5.0000, 7.5000, + 10.0, 25.0, 50.0, 75.0, + 100.0, INF +) METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'emulated'}) +METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('GetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class EmulatedDriver(_Driver): def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 5ecf613e2026aaf7bf016fd23329e591d540f9eb..4aa42b180d9816a9ecdf37a1ec351cb52b9ba41c 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -21,7 +21,7 @@ from apscheduler.job import Job from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.background import BackgroundScheduler from ncclient.manager import Manager, connect_ssh -from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF from common.tools.client.RetryDecorator import delay_exponential from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float from device.service.driver_api.Exceptions import UnsupportedResourceKeyException @@ -223,7 +223,23 @@ def edit_config( results[i] = e # if validation fails, store the exception return results +HISTOGRAM_BUCKETS = ( + # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF + 0.0001, 0.00025, 0.00050, 0.00075, + 0.0010, 0.0025, 0.0050, 0.0075, + 0.0100, 0.0250, 0.0500, 0.0750, + 0.1000, 0.2500, 0.5000, 0.7500, + 1.0000, 2.5000, 5.0000, 7.5000, + 10.0, 25.0, 50.0, 75.0, + 100.0, INF +) METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'openconfig'}) +METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('GetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class OpenConfigDriver(_Driver): def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py index 69e61db41639c819d158c4dd3d2519012fd48724..bc628c160eaaa9ac282c81bd4c0e02536e88a80c 100644 --- a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py @@ -14,7 +14,7 @@ import anytree, json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type @@ -25,7 +25,22 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) +HISTOGRAM_BUCKETS = ( + # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF + 0.0010, 0.0025, 0.0050, 0.0075, + 0.0100, 0.0250, 0.0500, 0.0750, + 0.1000, 0.2500, 0.5000, 0.7500, + 1.0000, 2.5000, 5.0000, 7.5000, + 10.0000, 25.000, 50.0000, 75.000, + 100.0, INF +) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l2nm_emulated'}) +METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L2NMEmulatedServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py index dca5942ace276a083df9fa3113f7dc3a5e56b3b3..f161225192dfe7f9eb0804b9d9bff4e5acba9e21 100644 --- a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py @@ -14,7 +14,7 @@ import anytree, json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type @@ -25,7 +25,22 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) +HISTOGRAM_BUCKETS = ( + # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF + 0.0010, 0.0025, 0.0050, 0.0075, + 0.0100, 0.0250, 0.0500, 0.0750, + 0.1000, 0.2500, 0.5000, 0.7500, + 1.0000, 2.5000, 5.0000, 7.5000, + 10.0000, 25.000, 50.0000, 75.000, + 100.0, INF +) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l3nm_emulated'}) +METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L3NMEmulatedServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py index 76a3357454f037d55ec6652e954a47195759ae2a..0f5cb6c558c1515b81d011074ecda7e167c47e90 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py @@ -14,7 +14,7 @@ import anytree, json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF from common.proto.context_pb2 import ConfigActionEnum, ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_length, chk_type @@ -25,7 +25,22 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) +HISTOGRAM_BUCKETS = ( + # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF + 0.0010, 0.0025, 0.0050, 0.0075, + 0.0100, 0.0250, 0.0500, 0.0750, + 0.1000, 0.2500, 0.5000, 0.7500, + 1.0000, 2.5000, 5.0000, 7.5000, + 10.0000, 25.000, 50.0000, 75.000, + 100.0, INF +) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l3nm_openconfig'}) +METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) +METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L3NMOpenConfigServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called