Merge branch 'feat/monitoring' into 'develop' (c1b33911) · Commits · TFS / controller

common_requirements.in

+1 −0

Original line number	Diff line number	Diff line
		@@ -6,3 +6,4 @@ prometheus-client==0.13.0
		protobuf==3.20.*
		pytest==6.2.5
		pytest-benchmark==3.4.1
		python-dateutil==2.8.2

manifests/monitoringservice.yaml

+25 −14

Original line number	Diff line number	Diff line
		@@ -29,19 +29,23 @@ spec:
		terminationGracePeriodSeconds: 5
		restartPolicy: Always
		containers:
		- name: influxdb
		image: influxdb:1.8
		- name: metricsdb
		image: questdb/questdb
		ports:
		- containerPort: 8086
		envFrom:
		- secretRef:
		name: influxdb-secrets
		- containerPort: 9000
		- containerPort: 9009
		- containerPort: 9003
		env:
		- name: QDB_CAIRO_COMMIT_LAG
		value: "1000"
		- name: QDB_CAIRO_MAX_UNCOMMITTED_ROWS
		value: "100000"
		readinessProbe:
		exec:
		command: ["curl", "-XGET", "localhost:8086/health"]
		command: ["curl", "-XGET", "localhost:9000"]
		livenessProbe:
		exec:
		command: ["curl", "-XGET", "localhost:8086/health"]
		command: ["curl", "-XGET", "localhost:9003/metrics"]
		resources:
		requests:
		cpu: 250m
		@@ -54,9 +58,15 @@ spec:
		imagePullPolicy: Always
		ports:
		- containerPort: 7070
		envFrom:
		- secretRef:
		name: monitoring-secrets
		env:
		- name: METRICSDB_HOSTNAME
		value: "localhost"
		- name: METRICSDB_ILP_PORT
		value: "9009"
		- name: METRICSDB_REST_PORT
		value: "9000"
		- name: METRICSDB_TABLE
		value: "monitoring"
		readinessProbe:
		exec:
		command: ["/bin/grpc_health_probe", "-addr=:7070"]
		@@ -70,6 +80,7 @@ spec:
		limits:
		cpu: 700m
		memory: 1024Mi

		---
		apiVersion: v1
		kind: Service
		@@ -84,7 +95,7 @@ spec:
		protocol: TCP
		port: 7070
		targetPort: 7070
		- name: influxdb
		- name: questdb
		protocol: TCP
		port: 8086
		targetPort: 8086
		port: 9000
		targetPort: 9000
		No newline at end of file

proto/monitoring.proto

+74 −79

Original line number	Diff line number	Diff line
		@@ -19,48 +19,27 @@ import "context.proto";
		import "kpi_sample_types.proto";

		service MonitoringService {
		rpc CreateKpi (KpiDescriptor ) returns (KpiId ) {}
		rpc EditKpiDescriptor (EditedKpiDescriptor ) returns (context.Empty ) {}
		rpc SetKpi (KpiDescriptor ) returns (KpiId ) {}
		rpc DeleteKpi (KpiId ) returns (context.Empty ) {}
		rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList) {}
		rpc CreateBundleKpi (BundleKpiDescriptor ) returns (KpiId ) {}
		rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {}
		rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {}
		rpc IncludeKpi (Kpi ) returns (context.Empty ) {}
		rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {}
		rpc QueryKpiData (KpiQuery ) returns (KpiList ) {}
		rpc SubscribeKpi (SubsDescriptor ) returns (stream KpiList ) {}
		rpc SetKpiSubscription (SubsDescriptor ) returns (stream KpiList ) {}
		rpc GetSubsDescriptor (SubscriptionID ) returns (SubsDescriptor ) {}
		rpc GetSubscriptions (context.Empty ) returns (SubsIDList ) {}
		rpc EditKpiSubscription (SubsDescriptor ) returns (context.Empty ) {}
		rpc CreateKpiAlarm (AlarmDescriptor ) returns ( AlarmID ) {}
		rpc EditKpiAlarm (AlarmDescriptor ) returns (context.Empty ) {}
		rpc DeleteSubscription (SubscriptionID ) returns (context.Empty ) {}
		rpc SetKpiAlarm (AlarmDescriptor ) returns (AlarmID ) {}
		rpc GetAlarms (context.Empty ) returns (AlarmIDList ) {}
		rpc GetAlarmDescriptor (AlarmID ) returns (AlarmDescriptor ) {}
		rpc GetAlarmResponseStream (AlarmID ) returns (stream AlarmResponse ) {}
		// rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {}
		// rpc GetInstantKpi (KpiId ) returns (KpiList ) {}
		rpc GetAlarmResponseStream(AlarmSubscription ) returns (stream AlarmResponse) {}
		rpc DeleteAlarm (AlarmID ) returns (context.Empty ) {}
		rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {}
		rpc GetInstantKpi (KpiId ) returns (KpiList ) {}
		}

		message KpiDescriptor {
		string kpi_description = 1;
		kpi_sample_types.KpiSampleType kpi_sample_type = 2;
		context.DeviceId device_id = 3;
		context.EndPointId endpoint_id = 4;
		context.ServiceId service_id = 5;
		context.SliceId slice_id = 6;
		}

		message BundleKpiDescriptor {
		string kpi_description = 1;
		repeated KpiId kpi_id_list = 2;
		kpi_sample_types.KpiSampleType kpi_sample_type = 3;
		context.DeviceId device_id = 4;
		context.EndPointId endpoint_id = 5;
		context.ServiceId service_id = 6;
		context.SliceId slice_id = 7;
		}

		message EditedKpiDescriptor {
		KpiId kpi_id = 1;
		string kpi_description = 2;
		repeated KpiId kpi_id_list = 3;
		@@ -83,8 +62,8 @@ message KpiQuery {
		float monitoring_window_s = 2;
		float sampling_rate_s = 3;
		uint32 last_n_samples = 4; // used when you want something like "get the last N many samples
		string start_date = 5; // used when you want something like "get the samples since X date/time"
		string end_date = 6; // used when you want something like "get the samples until X date/time"
		context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time"
		context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time"
		// Pending add field to reflect Available Device Protocols
		}

		@@ -94,24 +73,31 @@ message KpiId {

		message Kpi {
		KpiId kpi_id = 1;
		string timestamp = 2;
		context.Timestamp timestamp = 2;
		KpiValue kpi_value = 3;
		}

		message KpiValueRange {
		KpiValue kpiMinValue = 1;
		KpiValue kpiMaxValue = 2;
		bool inRange = 3; // by default True
		bool includeMinValue = 4; // False is outside the interval
		bool includeMaxValue = 5; // False is outside the interval
		}

		message KpiValue {
		oneof value {
		uint32 intVal = 1;
		float floatVal = 2;
		string stringVal = 3;
		bool boolVal = 4;
		int32 int32Val = 1;
		uint32 uint32Val = 2;
		int64 int64Val = 3;
		uint64 uint64Val = 4;
		float floatVal = 5;
		string stringVal = 6;
		bool boolVal = 7;
		}
		}


		message KpiList {
		repeated Kpi kpi_list = 1;
		}
		@@ -121,11 +107,12 @@ message KpiDescriptorList {
		}

		message SubsDescriptor{
		KpiId kpi_id = 1;
		float sampling_duration_s = 2;
		float sampling_interval_s = 3;
		string start_date = 4; // used when you want something like "get the samples since X date/time"
		string end_date = 5; // used when you want something like "get the samples until X date/time"
		SubscriptionID subs_id = 1;
		KpiId kpi_id = 2;
		float sampling_duration_s = 3;
		float sampling_interval_s = 4;
		context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time"
		context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time"
		// Pending add field to reflect Available Device Protocols
		}

		@@ -143,21 +130,29 @@ message SubsIDList {
		}

		message AlarmDescriptor {
		string alarm_description = 1;
		string name = 2;
		KpiId kpi_id = 3;
		KpiValueRange kpi_value_range = 4;
		string timestamp = 5;
		AlarmID alarm_id = 1;
		string alarm_description = 2;
		string name = 3;
		repeated KpiId kpi_id = 4;
		repeated KpiValueRange kpi_value_range = 5;
		context.Timestamp timestamp = 6;
		}

		message AlarmID{
		context.Uuid alarm_id = 1;
		}

		message AlarmSubscription{
		AlarmID alarmID = 1;
		float subscription_timeout_s = 2;
		float subscription_frequency_ms = 3;
		}

		message AlarmResponse {
		AlarmID alarm_id = 1;
		string text = 2;
		KpiValue kpi_value = 3;
		context.Timestamp timestamp = 4;
		}

		message AlarmIDList {

src/monitoring/Config.py→src/common/tools/timestamp/Converters.py

+8 −22

Original line number	Diff line number	Diff line
		@@ -12,29 +12,15 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import logging

		# General settings
		LOG_LEVEL = logging.WARNING

		# gRPC settings
		GRPC_SERVICE_PORT = 7070
		GRPC_MAX_WORKERS = 10
		GRPC_GRACE_PERIOD = 60
		GRPC_SERVICE_HOST = '127.0.0.1'

		# Prometheus settings
		METRICS_PORT = 9192

		# Dependency micro-service connection settings
		CONTEXT_SERVICE_HOST = '127.0.0.1'
		CONTEXT_GRPC_SERVICE_PORT = 1010

		DEVICE_SERVICE_HOST = '127.0.0.1'
		DEVICE_GRPC_SERVICE_PORT = 2020
		DEVICE_GRPC_MAX_WORKERS = 10
		DEVICE_GRPC_GRACE_PERIOD = 60

		import dateutil.parser
		from datetime import datetime, timezone

		def timestamp_string_to_float(str_timestamp : str) -> float:
		return datetime.timestamp(dateutil.parser.isoparse(str_timestamp))

		def timestamp_float_to_string(flt_timestamp : float) -> str:
		return datetime.utcfromtimestamp(flt_timestamp).isoformat() + 'Z'

		def timestamp_utcnow_to_float() -> float:
		return datetime.timestamp(datetime.now(tz=timezone.utc))

src/common/tools/timestamp/init.py

0 → 100644

+14 −0

Original line number	Diff line number	Diff line
		# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.