Merge branch 'feat/load-gen-add-metrics' into 'develop' (8d6d446c) · Commits · TFS / controller

deploy/tfs.sh

+9 −0

Original line number	Original line	Diff line number	Diff line
	@@ -495,5 +495,14 @@ if [[ "$TFS_COMPONENTS" == "webui" ]]; then
	curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}		curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
	echo		echo

			# Dashboard: Load Generator Status
			curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_prom_load_generator.json' \
			${GRAFANA_URL_UPDATED}/api/dashboards/db
			echo
			DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-loadgen-stats"
			DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" \| jq '.dashboard.id')
			curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
			echo

	printf "\n\n"		printf "\n\n"
	fi		fi

manifests/load_generatorservice.yaml

+5 −0

Original line number	Original line	Diff line number	Diff line
	@@ -33,6 +33,7 @@ spec:
	imagePullPolicy: Always		imagePullPolicy: Always
	ports:		ports:
	- containerPort: 50052		- containerPort: 50052
			- containerPort: 9192
	env:		env:
	- name: LOG_LEVEL		- name: LOG_LEVEL
	value: "INFO"		value: "INFO"
	@@ -65,3 +66,7 @@ spec:
	protocol: TCP		protocol: TCP
	port: 50052		port: 50052
	targetPort: 50052		targetPort: 50052
			- name: metrics
			protocol: TCP
			port: 9192
			targetPort: 9192

manifests/servicemonitors.yaml

+29 −0

Original line number	Original line	Diff line number	Diff line
	@@ -330,3 +330,32 @@ spec:
	any: false		any: false
	matchNames:		matchNames:
	- tfs # namespace where the app is running		- tfs # namespace where the app is running
			---
			apiVersion: monitoring.coreos.com/v1
			kind: ServiceMonitor
			metadata:
			namespace: monitoring # namespace where prometheus is running
			name: tfs-load-generatorservice-metric
			labels:
			app: load-generatorservice
			#release: prometheus
			#release: prom # name of the release
			# ( VERY IMPORTANT: You need to know the correct release name by viewing
			# the servicemonitor of Prometheus itself: Without the correct name,
			# Prometheus cannot identify the metrics of the Flask app as the target.)
			spec:
			selector:
			matchLabels:
			# Target app service
			#namespace: tfs
			app: load-generatorservice # same as above
			#release: prometheus # same as above
			endpoints:
			- port: metrics # named port in target app
			scheme: http
			path: /metrics # path to scrape
			interval: 5s # scrape interval
			namespaceSelector:
			any: false
			matchNames:
			- tfs # namespace where the app is running

proto/load_generator.proto

+20 −4

Original line number	Original line	Diff line number	Diff line
	@@ -33,16 +33,32 @@ enum RequestTypeEnum {
	REQUESTTYPE_SLICE_L3NM = 6;		REQUESTTYPE_SLICE_L3NM = 6;
	}		}

			message Range {
			float minimum = 1;
			float maximum = 2;
			}

			message ScalarOrRange {
			oneof value {
			float scalar = 1; // select the scalar value
			Range range = 2; // select a random uniformly dstributed value between minimum and maximum
			}
			}

	message Parameters {		message Parameters {
	uint64 num_requests = 1; // if == 0, generate infinite requests		uint64 num_requests = 1; // if == 0, generate infinite requests
	repeated RequestTypeEnum request_types = 2;		repeated RequestTypeEnum request_types = 2;
	float offered_load = 3;		float offered_load = 3;
	float holding_time = 4;		float holding_time = 4;
	float inter_arrival_time = 5;		float inter_arrival_time = 5;
	bool do_teardown = 6;		repeated ScalarOrRange availability = 6; // one from the list is selected
	bool dry_mode = 7;		repeated ScalarOrRange capacity_gbps = 7; // one from the list is selected
	bool record_to_dlt = 8;		repeated ScalarOrRange e2e_latency_ms = 8; // one from the list is selected
	string dlt_domain_id = 9;		uint32 max_workers = 9;
			bool do_teardown = 10;
			bool dry_mode = 11;
			bool record_to_dlt = 12;
			string dlt_domain_id = 13;
	}		}

	message Status {		message Status {

src/common/method_wrappers/Decorator.py

+33 −7

Original line number	Original line	Diff line number	Diff line
	@@ -15,7 +15,7 @@
	import grpc, json, logging, threading		import grpc, json, logging, threading
	from enum import Enum		from enum import Enum
	from prettytable import PrettyTable		from prettytable import PrettyTable
	from typing import Any, Dict, List, Set, Tuple		from typing import Any, Dict, List, Optional, Set, Tuple
	from prometheus_client import Counter, Histogram		from prometheus_client import Counter, Histogram
	from prometheus_client.metrics import MetricWrapperBase, INF		from prometheus_client.metrics import MetricWrapperBase, INF
	from common.tools.grpc.Tools import grpc_message_to_json_string		from common.tools.grpc.Tools import grpc_message_to_json_string
	@@ -25,12 +25,14 @@ class MetricTypeEnum(Enum):
	COUNTER_STARTED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_started'		COUNTER_STARTED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_started'
	COUNTER_COMPLETED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_completed'		COUNTER_COMPLETED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_completed'
	COUNTER_FAILED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_failed'		COUNTER_FAILED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_failed'
			COUNTER_BLOCKED = 'tfs_{component:s}_{sub_module:s}_{method:s}_counter_requests_blocked'
	HISTOGRAM_DURATION = 'tfs_{component:s}_{sub_module:s}_{method:s}_histogram_duration'		HISTOGRAM_DURATION = 'tfs_{component:s}_{sub_module:s}_{method:s}_histogram_duration'

	METRIC_TO_CLASS_PARAMS = {		METRIC_TO_CLASS_PARAMS = {
	MetricTypeEnum.COUNTER_STARTED : (Counter, {}),		MetricTypeEnum.COUNTER_STARTED : (Counter, {}),
	MetricTypeEnum.COUNTER_COMPLETED : (Counter, {}),		MetricTypeEnum.COUNTER_COMPLETED : (Counter, {}),
	MetricTypeEnum.COUNTER_FAILED : (Counter, {}),		MetricTypeEnum.COUNTER_FAILED : (Counter, {}),
			MetricTypeEnum.COUNTER_BLOCKED : (Counter, {}),
	MetricTypeEnum.HISTOGRAM_DURATION: (Histogram, {		MetricTypeEnum.HISTOGRAM_DURATION: (Histogram, {
	'buckets': (		'buckets': (
	# .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF		# .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF
	@@ -75,21 +77,45 @@ class MetricsPool:
	return MetricsPool.metrics[metric_name]		return MetricsPool.metrics[metric_name]

	def get_metrics(		def get_metrics(
	self, method : str		self, method : str, labels : Optional[Dict[str, str]] = None
	) -> Tuple[Histogram, Counter, Counter, Counter]:		) -> Tuple[Histogram, Counter, Counter, Counter]:
	histogram_duration : Histogram = self.get_or_create(method, MetricTypeEnum.HISTOGRAM_DURATION)		histogram_duration : Histogram = self.get_or_create(method, MetricTypeEnum.HISTOGRAM_DURATION)
	counter_started : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_STARTED)		counter_started : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_STARTED)
	counter_completed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_COMPLETED)		counter_completed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_COMPLETED)
	counter_failed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_FAILED)		counter_failed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_FAILED)

	if len(self._labels) > 0:		if labels is None and len(self._labels) > 0:
	histogram_duration = histogram_duration.labels(**(self._labels))		labels = self._labels
	counter_started = counter_started.labels(**(self._labels))
	counter_completed = counter_completed.labels(**(self._labels))		if labels is not None and len(labels) > 0:
	counter_failed = counter_failed.labels(**(self._labels))		histogram_duration = histogram_duration.labels(**labels)
			counter_started = counter_started.labels(**labels)
			counter_completed = counter_completed.labels(**labels)
			counter_failed = counter_failed.labels(**labels)

	return histogram_duration, counter_started, counter_completed, counter_failed		return histogram_duration, counter_started, counter_completed, counter_failed

			def get_metrics_loadgen(
			self, method : str, labels : Optional[Dict[str, str]] = None
			) -> Tuple[Histogram, Counter, Counter, Counter, Counter]:
			histogram_duration : Histogram = self.get_or_create(method, MetricTypeEnum.HISTOGRAM_DURATION)
			counter_started : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_STARTED)
			counter_completed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_COMPLETED)
			counter_failed : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_FAILED)
			counter_blocked : Counter = self.get_or_create(method, MetricTypeEnum.COUNTER_BLOCKED)

			if labels is None and len(self._labels) > 0:
			labels = self._labels

			if labels is not None and len(labels) > 0:
			histogram_duration = histogram_duration.labels(**labels)
			counter_started = counter_started.labels(**labels)
			counter_completed = counter_completed.labels(**labels)
			counter_failed = counter_failed.labels(**labels)
			counter_blocked = counter_blocked.labels(**labels)

			return histogram_duration, counter_started, counter_completed, counter_failed, counter_blocked

	def get_pretty_table(self, remove_empty_buckets : bool = True) -> PrettyTable:		def get_pretty_table(self, remove_empty_buckets : bool = True) -> PrettyTable:
	with MetricsPool.lock:		with MetricsPool.lock:
	method_to_metric_fields : Dict[str, Dict[str, Dict[str, Any]]] = dict()		method_to_metric_fields : Dict[str, Dict[str, Dict[str, Any]]] = dict()