Improve QueryKpiData RPC (86951ab9) · Commits · TFS / controller

proto/monitoring.proto

+7 −9

Original line number	Original line	Diff line number	Diff line
	@@ -59,13 +59,11 @@ message MonitorKpiRequest {
	}		}

	message KpiQuery {		message KpiQuery {
	repeated KpiId kpi_id = 1;		repeated KpiId kpi_ids = 1;
	float monitoring_window_s = 2;		float monitoring_window_s = 2;
	float sampling_rate_s = 3;		uint32 last_n_samples = 3; // used when you want something like "get the last N many samples
	uint32 last_n_samples = 4; // used when you want something like "get the last N many samples		context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time"
	context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time"		context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time"
	context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time"
	// Pending add field to reflect Available Device Protocols
	}		}


	@@ -76,11 +74,11 @@ message RawKpi { // cell

	message RawKpiList { // column		message RawKpiList { // column
	KpiId kpi_id = 1;		KpiId kpi_id = 1;
	repeated RawKpi raw_kpi_list = 2;		repeated RawKpi raw_kpis = 2;
	}		}

	message RawKpiTable { // table		message RawKpiTable { // table
	repeated RawKpiList raw_kpi_table = 1;		repeated RawKpiList raw_kpi_lists = 1;
	}		}

	message KpiId {		message KpiId {

src/monitoring/service/MetricsDBTools.py

+28 −6

Original line number	Original line	Diff line number	Diff line
	@@ -23,7 +23,7 @@ import datetime
	from common.tools.timestamp.Converters import timestamp_float_to_string, timestamp_utcnow_to_float		from common.tools.timestamp.Converters import timestamp_float_to_string, timestamp_utcnow_to_float
	import psycopg2		import psycopg2

	LOGGER = logging.getLogger(__name__)		from monitoring.service.MonitoringServiceServicerImpl import LOGGER


	class MetricsDB():		class MetricsDB():
	@@ -174,11 +174,33 @@ class MetricsDB():
	if connection:		if connection:
	connection.close()		connection.close()

	def get_raw_kpi_list(self, kpi_id, monitoring_window_s,sampling_rate_s, last_n_samples, start_timestamp, end_timestamp):		def get_raw_kpi_list(self, kpi_id, monitoring_window_s, last_n_samples, start_timestamp, end_timestamp):
	try:		try:
	end_date = timestamp_utcnow_to_float() - self.commit_lag_ms / 1000		query_root = f"SELECT timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' "
			query = query_root
			start_date = float()
			end_date = float()
			if last_n_samples:
			query = query + f"ORDER BY timestamp DESC limit {last_n_samples}"
			elif monitoring_window_s or start_timestamp or end_timestamp:
			if start_timestamp and end_timestamp:
			start_date = start_timestamp
			end_date = end_timestamp
			elif monitoring_window_s:
			if start_timestamp and not end_timestamp:
			start_date = start_timestamp
			end_date = start_date + monitoring_window_s
			elif end_timestamp and not start_timestamp:
			end_date = end_timestamp
			start_date = end_date - monitoring_window_s
			elif not start_timestamp and not end_timestamp:
			end_date = timestamp_utcnow_to_float()
	start_date = end_date - monitoring_window_s		start_date = end_date - monitoring_window_s
	query = f"SELECT timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"		query = query + f"AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"
			else:
			LOGGER.debug(f"Wrong parameters settings")

			LOGGER.debug(query)

	if self.postgre:		if self.postgre:
	kpi_list = self.run_query_postgre(query)		kpi_list = self.run_query_postgre(query)
	@@ -186,7 +208,6 @@ class MetricsDB():
	else:		else:
	kpi_list = self.run_query(query)		kpi_list = self.run_query(query)
	LOGGER.debug(f"kpi_list influx: {kpi_list}")		LOGGER.debug(f"kpi_list influx: {kpi_list}")

	if kpi_list:		if kpi_list:
	LOGGER.debug(f"New data received for subscription to KPI {kpi_id}")		LOGGER.debug(f"New data received for subscription to KPI {kpi_id}")
	return kpi_list		return kpi_list
	@@ -200,6 +221,7 @@ class MetricsDB():
	end_date = timestamp_utcnow_to_float() - self.commit_lag_ms / 1000		end_date = timestamp_utcnow_to_float() - self.commit_lag_ms / 1000
	start_date = end_date - sampling_interval_s		start_date = end_date - sampling_interval_s
	query = f"SELECT kpi_id, timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"		query = f"SELECT kpi_id, timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"
			LOGGER.debug(query)
	if self.postgre:		if self.postgre:
	kpi_list = self.run_query_postgre(query)		kpi_list = self.run_query_postgre(query)
	LOGGER.debug(f"kpi_list postgre: {kpi_list}")		LOGGER.debug(f"kpi_list postgre: {kpi_list}")

src/monitoring/service/MonitoringServiceServicerImpl.py

+6 −7

Original line number	Original line	Diff line number	Diff line
	@@ -251,9 +251,8 @@ class MonitoringServiceServicerImpl(MonitoringServiceServicer):

	LOGGER.debug(str(request))		LOGGER.debug(str(request))

	kpi_id_list = request.kpi_id		kpi_id_list = request.kpi_ids
	monitoring_window_s = request.monitoring_window_s		monitoring_window_s = request.monitoring_window_s
	sampling_rate_s = request.sampling_rate_s
	last_n_samples = request.last_n_samples		last_n_samples = request.last_n_samples
	start_timestamp = request.start_timestamp.timestamp		start_timestamp = request.start_timestamp.timestamp
	end_timestamp = request.end_timestamp.timestamp		end_timestamp = request.end_timestamp.timestamp
	@@ -268,7 +267,7 @@ class MonitoringServiceServicerImpl(MonitoringServiceServicer):
	break		break
	else:		else:
	# Execute query per Kpi_id and introduce their kpi_list in the table		# Execute query per Kpi_id and introduce their kpi_list in the table
	kpi_list = self.metrics_db.get_raw_kpi_list(kpi_id,monitoring_window_s,sampling_rate_s,last_n_samples,start_timestamp,end_timestamp)		kpi_list = self.metrics_db.get_raw_kpi_list(kpi_id,monitoring_window_s,last_n_samples,start_timestamp,end_timestamp)
	raw_kpi_list = RawKpiList()		raw_kpi_list = RawKpiList()
	raw_kpi_list.kpi_id.kpi_id.uuid = kpi_id		raw_kpi_list.kpi_id.kpi_id.uuid = kpi_id

	@@ -279,11 +278,11 @@ class MonitoringServiceServicerImpl(MonitoringServiceServicer):
	else:		else:
	for item in kpi_list:		for item in kpi_list:
	raw_kpi = RawKpi()		raw_kpi = RawKpi()
	raw_kpi.timestamp.timestamp = timestamp_string_to_float(item[1])		raw_kpi.timestamp.timestamp = timestamp_string_to_float(item[0])
	raw_kpi.kpi_value.floatVal = item[2]		raw_kpi.kpi_value.floatVal = item[1]
	raw_kpi_list.raw_kpi_list.append(raw_kpi)		raw_kpi_list.raw_kpis.append(raw_kpi)

	raw_kpi_table.raw_kpi_table.append(raw_kpi_list)		raw_kpi_table.raw_kpi_lists.append(raw_kpi_list)

	return raw_kpi_table		return raw_kpi_table
	except ServiceException as e:		except ServiceException as e:

src/monitoring/tests/Messages.py

+14 −16

Original line number	Original line	Diff line number	Diff line
	@@ -22,15 +22,15 @@ def kpi_id():
	_kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member		_kpi_id.kpi_id.uuid = str(1) # pylint: disable=maybe-no-member
	return _kpi_id		return _kpi_id

	def create_kpi_request():		def create_kpi_request(kpi_id_str):
	_create_kpi_request = monitoring_pb2.KpiDescriptor()		_create_kpi_request = monitoring_pb2.KpiDescriptor()
	_create_kpi_request.kpi_description = 'KPI Description Test'		_create_kpi_request.kpi_description = 'KPI Description Test'
	_create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED		_create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED
	_create_kpi_request.device_id.device_uuid.uuid = 'DEV1' # pylint: disable=maybe-no-member		_create_kpi_request.device_id.device_uuid.uuid = 'DEV' + str(kpi_id_str)
	_create_kpi_request.service_id.service_uuid.uuid = 'SERV1' # pylint: disable=maybe-no-member		_create_kpi_request.service_id.service_uuid.uuid = 'SERV' + str(kpi_id_str)
	_create_kpi_request.slice_id.slice_uuid.uuid = 'SLC1' # pylint: disable=maybe-no-member		_create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + str(kpi_id_str)
	_create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END1' # pylint: disable=maybe-no-member		_create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + str(kpi_id_str)
	_create_kpi_request.connection_id.connection_uuid.uuid = 'CON1' # pylint: disable=maybe-no-member		_create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + str(kpi_id_str)
	return _create_kpi_request		return _create_kpi_request

	def create_kpi_request_b():		def create_kpi_request_b():
	@@ -77,20 +77,18 @@ def kpi_descriptor_list():
	def kpi_query(kpi_id_list):		def kpi_query(kpi_id_list):
	_kpi_query = monitoring_pb2.KpiQuery()		_kpi_query = monitoring_pb2.KpiQuery()

	_kpi_query.kpi_id.extend(kpi_id_list)		_kpi_query.kpi_ids.extend(kpi_id_list)
	_kpi_query.monitoring_window_s = 20		# _kpi_query.monitoring_window_s = 10
	_kpi_query.sampling_rate_s = 3		# _kpi_query.last_n_samples = 2
	# _kpi_query.last_n_samples = 10		_kpi_query.start_timestamp.timestamp = timestamp_utcnow_to_float() - 10
	_kpi_query.start_timestamp.timestamp = timestamp_utcnow_to_float() - 20
	_kpi_query.end_timestamp.timestamp = timestamp_utcnow_to_float()		_kpi_query.end_timestamp.timestamp = timestamp_utcnow_to_float()


	return _kpi_query		return _kpi_query

	def subs_descriptor(kpi_id):		def subs_descriptor(kpi_id):
	_subs_descriptor = monitoring_pb2.SubsDescriptor()		_subs_descriptor = monitoring_pb2.SubsDescriptor()

	sampling_duration_s = 20		sampling_duration_s = 10
	sampling_interval_s = 3		sampling_interval_s = 3
	real_start_time = timestamp_utcnow_to_float()		real_start_time = timestamp_utcnow_to_float()
	start_timestamp = real_start_time		start_timestamp = real_start_time