Merge branch 'feat/load-gen-add-metrics' into 'develop' (8d6d446c) · Commits · TFS / controller

deploy/tfs.sh

+9 −0

Original line number	Diff line number	Diff line
		@@ -495,5 +495,14 @@ if [[ "$TFS_COMPONENTS" == "webui" ]]; then
		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
		echo

		# Dashboard: Load Generator Status
		curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_prom_load_generator.json' \
		${GRAFANA_URL_UPDATED}/api/dashboards/db
		echo
		DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-loadgen-stats"
		DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')
		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
		echo

		printf "\n\n"
		fi

manifests/load_generatorservice.yaml

+5 −0

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ spec:
		imagePullPolicy: Always
		ports:
		- containerPort: 50052
		- containerPort: 9192
		env:
		- name: LOG_LEVEL
		value: "INFO"
		@@ -65,3 +66,7 @@ spec:
		protocol: TCP
		port: 50052
		targetPort: 50052
		- name: metrics
		protocol: TCP
		port: 9192
		targetPort: 9192

manifests/servicemonitors.yaml

+29 −0

Original line number	Diff line number	Diff line
		@@ -330,3 +330,32 @@ spec:
		any: false
		matchNames:
		- tfs # namespace where the app is running
		---
		apiVersion: monitoring.coreos.com/v1
		kind: ServiceMonitor
		metadata:
		namespace: monitoring # namespace where prometheus is running
		name: tfs-load-generatorservice-metric
		labels:
		app: load-generatorservice
		#release: prometheus
		#release: prom # name of the release
		# ( VERY IMPORTANT: You need to know the correct release name by viewing
		# the servicemonitor of Prometheus itself: Without the correct name,
		# Prometheus cannot identify the metrics of the Flask app as the target.)
		spec:
		selector:
		matchLabels:
		# Target app service
		#namespace: tfs
		app: load-generatorservice # same as above
		#release: prometheus # same as above
		endpoints:
		- port: metrics # named port in target app
		scheme: http
		path: /metrics # path to scrape
		interval: 5s # scrape interval
		namespaceSelector:
		any: false
		matchNames:
		- tfs # namespace where the app is running

proto/load_generator.proto

+20 −4

Original line number	Diff line number	Diff line
		@@ -33,16 +33,32 @@ enum RequestTypeEnum {
		REQUESTTYPE_SLICE_L3NM = 6;
		}

		message Range {
		float minimum = 1;
		float maximum = 2;
		}

		message ScalarOrRange {
		oneof value {
		float scalar = 1; // select the scalar value
		Range range = 2; // select a random uniformly dstributed value between minimum and maximum
		}
		}

		message Parameters {
		uint64 num_requests = 1; // if == 0, generate infinite requests
		repeated RequestTypeEnum request_types = 2;
		float offered_load = 3;
		float holding_time = 4;
		float inter_arrival_time = 5;
		bool do_teardown = 6;
		bool dry_mode = 7;
		bool record_to_dlt = 8;
		string dlt_domain_id = 9;
		repeated ScalarOrRange availability = 6; // one from the list is selected
		repeated ScalarOrRange capacity_gbps = 7; // one from the list is selected
		repeated ScalarOrRange e2e_latency_ms = 8; // one from the list is selected
		uint32 max_workers = 9;
		bool do_teardown = 10;
		bool dry_mode = 11;
		bool record_to_dlt = 12;
		string dlt_domain_id = 13;
		}

		message Status {

src/common/method_wrappers/Decorator.py

+33 −7

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@
		import grpc, json, logging, threading
		from enum import Enum
		from prettytable import PrettyTable
		from typing import Any, Dict, List, Set, Tuple
		from typing import Any, Dict, List, Optional, Set, Tuple
		from prometheus_client import Counter, Histogram
		from prometheus_client.metrics import MetricWrapperBase, INF
		from common.tools.grpc.Tools import grpc_message_to_json_string
		@@ -25,12 +25,14 @@ class MetricTypeEnum(Enum):
		COUNTER_STARTED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_started'
		COUNTER_COMPLETED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_completed'
		COUNTER_FAILED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_failed'
		COUNTER_BLOCKED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_blocked'
		HISTOGRAM_DURATION = 'tfs_{component:s}_{sub_module:s}_{method:s}_histogram_duration'

		METRIC_TO_CLASS_PARAMS = {
		MetricTypeEnum.COUNTER_STARTED : (Counter, {}),
		MetricTypeEnum.COUNTER_COMPLETED : (Counter, {}),
		MetricTypeEnum.COUNTER_FAILED : (Counter, {}),
		MetricTypeEnum.COUNTER_BLOCKED : (Counter, {}),
		MetricTypeEnum.HISTOGRAM_DURATION: (Histogram, {
		'buckets': (
		# .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF
		@@ -75,21 +77,45 @@ class MetricsPool:
		return MetricsPool.metrics[metric_name]

		def get_metrics(
		self, method : str
		self, method : str, labels : Optional[Dict[str, str]] = None
		) -> Tuple[Histogram, Counter, Counter, Counter]:
		histogram_duration : Histogram = self.get_or_create(method, MetricTypeEnum.HISTOGRAM_DURATION)
		counter_started : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_STARTED)
		counter_completed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_COMPLETED)
		counter_failed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_FAILED)

		if len(self._labels) > 0:
		histogram_duration = histogram_duration.labels(**(self._labels))
		counter_started = counter_started.labels(**(self._labels))
		counter_completed = counter_completed.labels(**(self._labels))
		counter_failed = counter_failed.labels(**(self._labels))
		if labels is None and len(self._labels) > 0:
		labels = self._labels

		if labels is not None and len(labels) > 0:
		histogram_duration = histogram_duration.labels(**labels)
		counter_started = counter_started.labels(**labels)
		counter_completed = counter_completed.labels(**labels)
		counter_failed = counter_failed.labels(**labels)

		return histogram_duration, counter_started, counter_completed, counter_failed

		def get_metrics_loadgen(
		self, method : str, labels : Optional[Dict[str, str]] = None
		) -> Tuple[Histogram, Counter, Counter, Counter, Counter]:
		histogram_duration : Histogram = self.get_or_create(method, MetricTypeEnum.HISTOGRAM_DURATION)
		counter_started : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_STARTED)
		counter_completed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_COMPLETED)
		counter_failed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_FAILED)
		counter_blocked : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_BLOCKED)

		if labels is None and len(self._labels) > 0:
		labels = self._labels

		if labels is not None and len(labels) > 0:
		histogram_duration = histogram_duration.labels(**labels)
		counter_started = counter_started.labels(**labels)
		counter_completed = counter_completed.labels(**labels)
		counter_failed = counter_failed.labels(**labels)
		counter_blocked = counter_blocked.labels(**labels)

		return histogram_duration, counter_started, counter_completed, counter_failed, counter_blocked

		def get_pretty_table(self, remove_empty_buckets : bool = True) -> PrettyTable:
		with MetricsPool.lock:
		method_to_metric_fields : Dict[str, Dict[str, Dict[str, Any]]] = dict()