From 2048e5ed816f06c806b1b7c2dc71756d95d80e5d Mon Sep 17 00:00:00 2001
From: Lluis Gifre <lluis.gifre@cttc.es>
Date: Thu, 21 Jul 2022 14:40:46 +0200
Subject: [PATCH] Improvements in Monitoring:
- Arranged commit before merge
- Replaced string-based timestamps by context.Timestamp
- Created helper functions to convert timestamps string from/to float
- Arranged requirements to meet dependencies of converter functions
---
common_requirements.in | 1 +
proto/monitoring.proto | 149 ++++++++++++-----------
proto/te.proto | 2 +-
src/common/proto | 1 +
src/common/tools/service/__init__.py | 14 +++
src/common/tools/timestamp/Converters.py | 11 ++
src/common/tools/timestamp/__init__.py | 14 +++
src/device/requirements.in | 1 -
8 files changed, 118 insertions(+), 75 deletions(-)
create mode 120000 src/common/proto
create mode 100644 src/common/tools/service/__init__.py
create mode 100644 src/common/tools/timestamp/Converters.py
create mode 100644 src/common/tools/timestamp/__init__.py
diff --git a/common_requirements.in b/common_requirements.in
index 8a027cfbd..772c1115d 100644
--- a/common_requirements.in
+++ b/common_requirements.in
@@ -6,3 +6,4 @@ prometheus-client==0.13.0
protobuf==3.20.*
pytest==6.2.5
pytest-benchmark==3.4.1
+python-dateutil==2.8.2
diff --git a/proto/monitoring.proto b/proto/monitoring.proto
index 0086cb43d..64d7bdbfe 100644
--- a/proto/monitoring.proto
+++ b/proto/monitoring.proto
@@ -19,56 +19,56 @@ import "context.proto";
import "kpi_sample_types.proto";
service MonitoringService {
- rpc CreateKpi (KpiDescriptor ) returns (KpiId ) {}
- rpc EditKpiDescriptor (EditedKpiDescriptor ) returns (context.Empty ) {}
- rpc DeleteKpi (KpiId ) returns (context.Empty ) {}
- rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {}
- rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList) {}
- rpc CreateBundleKpi (BundleKpiDescriptor ) returns (KpiId ) {}
- rpc IncludeKpi (Kpi ) returns (context.Empty ) {}
- rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {}
- rpc QueryKpiData (KpiQuery ) returns (KpiList ) {}
- rpc SubscribeKpi (SubsDescriptor ) returns (stream KpiList ) {}
- rpc GetSubsDescriptor (SubscriptionID ) returns (SubsDescriptor ) {}
- rpc GetSubscriptions (context.Empty ) returns (SubsIDList ) {}
- rpc EditKpiSubscription (SubsDescriptor ) returns (context.Empty ) {}
- rpc CreateKpiAlarm (AlarmDescriptor ) returns (AlarmID ) {}
- rpc EditKpiAlarm (AlarmDescriptor ) returns (context.Empty ) {}
- rpc GetAlarms (context.Empty ) returns (AlarmIDList ) {}
- rpc GetAlarmDescriptor (AlarmSubscription ) returns (AlarmDescriptor ) {}
- rpc GetAlarmResponseStream (AlarmID ) returns (stream AlarmResponse ) {}
- rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {}
- rpc GetInstantKpi (KpiId ) returns (KpiList ) {}
+ rpc CreateKpi (KpiDescriptor ) returns (KpiId ) {}
+ rpc EditKpiDescriptor (EditedKpiDescriptor) returns (context.Empty ) {}
+ rpc DeleteKpi (KpiId ) returns (context.Empty ) {}
+ rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {}
+ rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {}
+ rpc CreateBundleKpi (BundleKpiDescriptor) returns (KpiId ) {}
+ rpc IncludeKpi (Kpi ) returns (context.Empty ) {}
+ rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {}
+ rpc QueryKpiData (KpiQuery ) returns (KpiList ) {}
+ rpc SubscribeKpi (SubsDescriptor ) returns (stream KpiList ) {}
+ rpc GetSubsDescriptor (SubscriptionID ) returns (SubsDescriptor ) {}
+ rpc GetSubscriptions (context.Empty ) returns (SubsIDList ) {}
+ rpc EditKpiSubscription (SubsDescriptor ) returns (context.Empty ) {}
+ rpc CreateKpiAlarm (AlarmDescriptor ) returns (AlarmID ) {}
+ rpc EditKpiAlarm (AlarmDescriptor ) returns (context.Empty ) {}
+ rpc GetAlarms (context.Empty ) returns (AlarmIDList ) {}
+ rpc GetAlarmDescriptor (AlarmSubscription ) returns (AlarmDescriptor ) {}
+ rpc GetAlarmResponseStream(AlarmID ) returns (stream AlarmResponse) {}
+ rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {}
+ rpc GetInstantKpi (KpiId ) returns (KpiList ) {}
}
message KpiDescriptor {
- string kpi_description = 1;
- kpi_sample_types.KpiSampleType kpi_sample_type = 2;
- context.DeviceId device_id = 3;
- context.EndPointId endpoint_id = 4;
- context.ServiceId service_id = 5;
- context.SliceId slice_id = 6;
+ string kpi_description = 1;
+ kpi_sample_types.KpiSampleType kpi_sample_type = 2;
+ context.DeviceId device_id = 3;
+ context.EndPointId endpoint_id = 4;
+ context.ServiceId service_id = 5;
+ context.SliceId slice_id = 6;
}
message BundleKpiDescriptor {
- string kpi_description = 1;
- repeated KpiId kpi_id_list = 2;
- kpi_sample_types.KpiSampleType kpi_sample_type = 3;
- context.DeviceId device_id = 4;
- context.EndPointId endpoint_id = 5;
- context.ServiceId service_id = 6;
- context.SliceId slice_id = 7;
+ string kpi_description = 1;
+ repeated KpiId kpi_id_list = 2;
+ kpi_sample_types.KpiSampleType kpi_sample_type = 3;
+ context.DeviceId device_id = 4;
+ context.EndPointId endpoint_id = 5;
+ context.ServiceId service_id = 6;
+ context.SliceId slice_id = 7;
}
message EditedKpiDescriptor {
- KpiId kpi_id = 1;
- string kpi_description = 2;
- repeated KpiId kpi_id_list = 3;
- kpi_sample_types.KpiSampleType kpi_sample_type = 4;
- context.DeviceId device_id = 5;
- context.EndPointId endpoint_id = 6;
- context.ServiceId service_id = 7;
- context.SliceId slice_id = 8;
+ KpiId kpi_id = 1;
+ string kpi_description = 2;
+ repeated KpiId kpi_id_list = 3;
+ kpi_sample_types.KpiSampleType kpi_sample_type = 4;
+ context.DeviceId device_id = 5;
+ context.EndPointId endpoint_id = 6;
+ context.ServiceId service_id = 7;
+ context.SliceId slice_id = 8;
}
message MonitorKpiRequest {
@@ -79,12 +79,12 @@ message MonitorKpiRequest {
}
message KpiQuery {
- repeated KpiId kpi_id = 1;
- float monitoring_window_s = 2;
- float sampling_rate_s = 3;
- uint32 last_n_samples = 4; // used when you want something like "get the last N many samples
- string start_date = 5; // used when you want something like "get the samples since X date/time"
- string end_date = 6; // used when you want something like "get the samples until X date/time"
+ repeated KpiId kpi_id = 1;
+ float monitoring_window_s = 2;
+ float sampling_rate_s = 3;
+ uint32 last_n_samples = 4; // used when you want something like "get the last N many samples
+ context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time"
+ context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time"
// Pending add field to reflect Available Device Protocols
}
@@ -93,9 +93,9 @@ message KpiId {
}
message Kpi {
- KpiId kpi_id = 1;
- string timestamp = 2;
- KpiValue kpi_value = 3;
+ KpiId kpi_id = 1;
+ context.Timestamp timestamp = 2;
+ KpiValue kpi_value = 3;
}
message KpiValueRange {
@@ -106,27 +106,30 @@ message KpiValueRange {
message KpiValue {
oneof value {
- uint32 intVal = 1;
- float floatVal = 2;
- string stringVal = 3;
- bool boolVal = 4;
+ int32 int32Val = 1;
+ uint32 uint32Val = 2;
+ int64 int64Val = 3;
+ uint64 uint64Val = 4;
+ float floatVal = 5;
+ string stringVal = 6;
+ bool boolVal = 7;
}
}
message KpiList {
- repeated Kpi kpi_list = 1;
+ repeated Kpi kpi_list = 1;
}
message KpiDescriptorList {
- repeated KpiDescriptor kpi_descriptor_list = 1;
+ repeated KpiDescriptor kpi_descriptor_list = 1;
}
message SubsDescriptor{
- KpiId kpi_id = 1;
- float sampling_duration_s = 2;
- float sampling_interval_s = 3;
- string start_date = 4; // used when you want something like "get the samples since X date/time"
- string end_date = 5; // used when you want something like "get the samples until X date/time"
+ KpiId kpi_id = 1;
+ float sampling_duration_s = 2;
+ float sampling_interval_s = 3;
+ context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time"
+ context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time"
// Pending add field to reflect Available Device Protocols
}
@@ -135,7 +138,7 @@ message SubscriptionID {
}
message SubsResponse {
- SubscriptionID subs_id = 1;
+ SubscriptionID subs_id = 1;
repeated KpiList kpi_list = 2;
}
@@ -144,11 +147,11 @@ message SubsIDList {
}
message AlarmDescriptor {
- string alarm_description = 1;
- string name = 2;
- KpiId kpi_id = 3;
- KpiValueRange kpi_value_range = 4;
- context.Timestamp timestamp = 5; // Check context
+ string alarm_description = 1;
+ string name = 2;
+ KpiId kpi_id = 3;
+ KpiValueRange kpi_value_range = 4;
+ context.Timestamp timestamp = 5;
}
message AlarmID{
@@ -157,15 +160,15 @@ message AlarmID{
message AlarmSubscription{
AlarmID alarmID = 1;
- float subscription_timeout_s = 2;
- float subscription_frequency_ms = 3;
+ float subscription_timeout_s = 2;
+ float subscription_frequency_ms = 3;
}
message AlarmResponse {
- AlarmID alarm_id = 1;
- string text = 2;
- KpiValue kpi_value = 3;
- context.Timestamp timestamp = 4; // Check context
+ AlarmID alarm_id = 1;
+ string text = 2;
+ KpiValue kpi_value = 3;
+ context.Timestamp timestamp = 4;
}
message AlarmIDList {
diff --git a/proto/te.proto b/proto/te.proto
index e646da10b..d639cb7f4 100644
--- a/proto/te.proto
+++ b/proto/te.proto
@@ -20,5 +20,5 @@ import "context.proto";
service TEService {
rpc RequestLSP(context.Service ) returns (context.ServiceStatus) {}
rpc UpdateLSP (context.ServiceId) returns (context.ServiceStatus) {}
- rpc DeleteLSP (context.ServiceId) returns (context.Empty ) {}
+ rpc DeleteLSP (context.ServiceId) returns (context.Empty ) {}
}
diff --git a/src/common/proto b/src/common/proto
new file mode 120000
index 000000000..0ae252a78
--- /dev/null
+++ b/src/common/proto
@@ -0,0 +1 @@
+../../proto/src/python
\ No newline at end of file
diff --git a/src/common/tools/service/__init__.py b/src/common/tools/service/__init__.py
new file mode 100644
index 000000000..70a332512
--- /dev/null
+++ b/src/common/tools/service/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/src/common/tools/timestamp/Converters.py b/src/common/tools/timestamp/Converters.py
new file mode 100644
index 000000000..dc7399d24
--- /dev/null
+++ b/src/common/tools/timestamp/Converters.py
@@ -0,0 +1,11 @@
+import dateutil.parser
+from datetime import datetime, timezone
+
+def timestamp_string_to_float(str_timestamp : str) -> float:
+ return datetime.timestamp(dateutil.parser.isoparse(str_timestamp))
+
+def timestamp_float_to_string(flt_timestamp : float) -> str:
+ return datetime.utcfromtimestamp(flt_timestamp).isoformat() + 'Z'
+
+def timestamp_utcnow_to_float() -> float:
+ return datetime.timestamp(datetime.now(tz=timezone.utc))
diff --git a/src/common/tools/timestamp/__init__.py b/src/common/tools/timestamp/__init__.py
new file mode 100644
index 000000000..70a332512
--- /dev/null
+++ b/src/common/tools/timestamp/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/src/device/requirements.in b/src/device/requirements.in
index e5aaddecb..10506fbd4 100644
--- a/src/device/requirements.in
+++ b/src/device/requirements.in
@@ -5,7 +5,6 @@ Jinja2==3.0.3
ncclient==0.6.13
p4runtime==1.3.0
paramiko==2.9.2
-python-dateutil==2.8.2
python-json-logger==2.0.2
pytz==2021.3
redis==4.1.2
--
GitLab