Commit c1b33911 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/monitoring' into 'develop'

Modifications in the monitoring information model and skeleton of the methods...

See merge request teraflow-h2020/controller!163
parents ff63a54c a7af976e
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -6,3 +6,4 @@ prometheus-client==0.13.0
protobuf==3.20.*
pytest==6.2.5
pytest-benchmark==3.4.1
python-dateutil==2.8.2
+25 −14
Original line number Diff line number Diff line
@@ -29,19 +29,23 @@ spec:
      terminationGracePeriodSeconds: 5
      restartPolicy: Always
      containers:
      - name: influxdb
        image: influxdb:1.8
      - name: metricsdb
        image: questdb/questdb
        ports:
        - containerPort: 8086
        envFrom:
          - secretRef:
              name: influxdb-secrets
        - containerPort: 9000
        - containerPort: 9009
        - containerPort: 9003
        env:
        - name: QDB_CAIRO_COMMIT_LAG
          value: "1000"
        - name: QDB_CAIRO_MAX_UNCOMMITTED_ROWS
          value: "100000"
        readinessProbe:
          exec:
            command: ["curl", "-XGET", "localhost:8086/health"]
            command: ["curl", "-XGET", "localhost:9000"]
        livenessProbe:
          exec:
            command: ["curl", "-XGET", "localhost:8086/health"]
            command: ["curl", "-XGET", "localhost:9003/metrics"]
        resources:
          requests:
            cpu: 250m
@@ -54,9 +58,15 @@ spec:
        imagePullPolicy: Always
        ports:
        - containerPort: 7070
        envFrom:
          - secretRef:
              name: monitoring-secrets
        env:
        - name: METRICSDB_HOSTNAME
          value: "localhost"
        - name: METRICSDB_ILP_PORT
          value: "9009"
        - name: METRICSDB_REST_PORT
          value: "9000"
        - name: METRICSDB_TABLE
          value: "monitoring"
        readinessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:7070"]
@@ -70,6 +80,7 @@ spec:
          limits:
            cpu: 700m
            memory: 1024Mi

---
apiVersion: v1
kind: Service
@@ -84,7 +95,7 @@ spec:
    protocol: TCP
    port: 7070
    targetPort: 7070
  - name: influxdb
  - name: questdb
    protocol: TCP
    port: 8086
    targetPort: 8086
    port: 9000
    targetPort: 9000
 No newline at end of file
+74 −79
Original line number Diff line number Diff line
@@ -19,48 +19,27 @@ import "context.proto";
import "kpi_sample_types.proto";

service MonitoringService {
  rpc CreateKpi                (KpiDescriptor        )     returns (KpiId            ) {}
  rpc EditKpiDescriptor        (EditedKpiDescriptor  )     returns (context.Empty    ) {}
  rpc SetKpi                (KpiDescriptor      ) returns (KpiId               ) {}
  rpc DeleteKpi             (KpiId              ) returns (context.Empty       ) {}
  rpc GetKpiDescriptorList     (context.Empty        )     returns (KpiDescriptorList) {}
  rpc CreateBundleKpi          (BundleKpiDescriptor  )     returns (KpiId            ) {}
  rpc GetKpiDescriptor      (KpiId              ) returns (KpiDescriptor       ) {}
  rpc GetKpiDescriptorList  (context.Empty      ) returns (KpiDescriptorList   ) {}
  rpc IncludeKpi            (Kpi                ) returns (context.Empty       ) {}
  rpc MonitorKpi            (MonitorKpiRequest  ) returns (context.Empty       ) {}
  rpc QueryKpiData          (KpiQuery           ) returns (KpiList             ) {}
  rpc SubscribeKpi             (SubsDescriptor       )     returns (stream KpiList   ) {}
  rpc SetKpiSubscription    (SubsDescriptor     ) returns (stream KpiList      ) {}
  rpc GetSubsDescriptor     (SubscriptionID     ) returns (SubsDescriptor      ) {}
  rpc GetSubscriptions      (context.Empty      ) returns (SubsIDList          ) {}
  rpc EditKpiSubscription      (SubsDescriptor       )     returns (context.Empty    ) {}
  rpc CreateKpiAlarm           (AlarmDescriptor      )     returns ( AlarmID ) {}
  rpc EditKpiAlarm             (AlarmDescriptor      )     returns (context.Empty    ) {}
  rpc DeleteSubscription    (SubscriptionID     ) returns (context.Empty       ) {}
  rpc SetKpiAlarm           (AlarmDescriptor    ) returns (AlarmID             ) {}
  rpc GetAlarms             (context.Empty      ) returns (AlarmIDList         ) {}
  rpc GetAlarmDescriptor    (AlarmID            ) returns (AlarmDescriptor     ) {}
  rpc GetAlarmResponseStream   (AlarmID              )     returns (stream AlarmResponse ) {}
  //  rpc GetStreamKpi            (KpiId                )     returns (stream Kpi       ) {}
  //  rpc GetInstantKpi           (KpiId                )     returns (KpiList          ) {}
  rpc GetAlarmResponseStream(AlarmSubscription  ) returns (stream AlarmResponse) {}
  rpc DeleteAlarm           (AlarmID            ) returns (context.Empty       ) {}
  rpc GetStreamKpi          (KpiId              ) returns (stream Kpi          ) {}
  rpc GetInstantKpi         (KpiId              ) returns (KpiList             ) {}
}

message KpiDescriptor {
  string kpi_description                          = 1;
  kpi_sample_types.KpiSampleType kpi_sample_type  = 2;
  context.DeviceId device_id                      = 3;
  context.EndPointId endpoint_id                  = 4;
  context.ServiceId  service_id                   = 5;
  context.SliceId    slice_id                     = 6;
}

message BundleKpiDescriptor {
  string kpi_description                          = 1;
  repeated KpiId kpi_id_list                      = 2;
  kpi_sample_types.KpiSampleType kpi_sample_type  = 3;
  context.DeviceId device_id                      = 4;
  context.EndPointId endpoint_id                  = 5;
  context.ServiceId  service_id                   = 6;
  context.SliceId    slice_id                     = 7;
}

message EditedKpiDescriptor {
  KpiId                          kpi_id          = 1;
  string                         kpi_description = 2;
  repeated KpiId                 kpi_id_list     = 3;
@@ -83,8 +62,8 @@ message KpiQuery {
  float             monitoring_window_s = 2;
  float             sampling_rate_s     = 3;
  uint32            last_n_samples      = 4;  // used when you want something like "get the last N many samples
  string start_date         = 5;  // used when you want something like "get the samples since X date/time"
  string end_date           = 6;  // used when you want something like "get the samples until X date/time"
  context.Timestamp start_timestamp     = 5;  // used when you want something like "get the samples since X date/time"
  context.Timestamp end_timestamp       = 6;  // used when you want something like "get the samples until X date/time"
  // Pending add field to reflect Available Device Protocols
}

@@ -94,24 +73,31 @@ message KpiId {

message Kpi {
  KpiId             kpi_id    = 1;
  string timestamp    = 2;
  context.Timestamp timestamp = 2;
  KpiValue          kpi_value = 3;
}

message KpiValueRange {
  KpiValue  kpiMinValue     = 1;
  KpiValue  kpiMaxValue     = 2;
  bool      inRange         = 3;  // by default True
  bool      includeMinValue = 4;  // False is outside the interval
  bool      includeMaxValue = 5;  // False is outside the interval
}

message KpiValue {
  oneof value {
    uint32 intVal     = 1;
    float floatVal    = 2;
    string stringVal  = 3;
    bool boolVal      = 4;
    int32  int32Val  = 1;
    uint32 uint32Val = 2;
    int64  int64Val  = 3;
    uint64 uint64Val = 4;
    float  floatVal  = 5;
    string stringVal = 6;
    bool   boolVal   = 7;
  }
}


message KpiList {
  repeated Kpi kpi_list = 1;
}
@@ -121,11 +107,12 @@ message KpiDescriptorList {
}

message SubsDescriptor{
  KpiId kpi_id              = 1;
  float sampling_duration_s = 2;
  float sampling_interval_s = 3;
  string start_date         = 4;  // used when you want something like "get the samples since X date/time"
  string end_date           = 5;  // used when you want something like "get the samples until X date/time"
  SubscriptionID    subs_id             = 1;
  KpiId             kpi_id              = 2;
  float             sampling_duration_s = 3;
  float             sampling_interval_s = 4;
  context.Timestamp start_timestamp     = 5;  // used when you want something like "get the samples since X date/time"
  context.Timestamp end_timestamp       = 6;  // used when you want something like "get the samples until X date/time"
  // Pending add field to reflect Available Device Protocols
}

@@ -143,21 +130,29 @@ message SubsIDList {
}

message AlarmDescriptor {
  string alarm_description      = 1;
  string name                   = 2;
  KpiId kpi_id                  = 3;
  KpiValueRange kpi_value_range = 4;
  string timestamp              = 5;
  AlarmID                     alarm_id              = 1;
  string                      alarm_description     = 2;
  string                      name                  = 3;
  repeated KpiId              kpi_id                = 4;
  repeated KpiValueRange      kpi_value_range       = 5;
  context.Timestamp           timestamp             = 6;
}

message AlarmID{
  context.Uuid alarm_id = 1;
}

message AlarmSubscription{
  AlarmID alarmID                   = 1;
  float   subscription_timeout_s    = 2;
  float   subscription_frequency_ms = 3;
}

message AlarmResponse {
  AlarmID           alarm_id  = 1;
  string            text      = 2;
  KpiValue          kpi_value = 3;
  context.Timestamp timestamp = 4;
}

message AlarmIDList {
+8 −22
Original line number Diff line number Diff line
@@ -12,29 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

# General settings
LOG_LEVEL = logging.WARNING

# gRPC settings
GRPC_SERVICE_PORT = 7070
GRPC_MAX_WORKERS  = 10
GRPC_GRACE_PERIOD = 60
GRPC_SERVICE_HOST = '127.0.0.1'

# Prometheus settings
METRICS_PORT = 9192

# Dependency micro-service connection settings
CONTEXT_SERVICE_HOST = '127.0.0.1'
CONTEXT_GRPC_SERVICE_PORT = 1010

DEVICE_SERVICE_HOST         = '127.0.0.1'
DEVICE_GRPC_SERVICE_PORT    = 2020
DEVICE_GRPC_MAX_WORKERS     = 10
DEVICE_GRPC_GRACE_PERIOD    = 60

import dateutil.parser
from datetime import datetime, timezone

def timestamp_string_to_float(str_timestamp : str) -> float:
    return datetime.timestamp(dateutil.parser.isoparse(str_timestamp))

def timestamp_float_to_string(flt_timestamp : float) -> str:
    return datetime.utcfromtimestamp(flt_timestamp).isoformat() + 'Z'

def timestamp_utcnow_to_float() -> float:
    return datetime.timestamp(datetime.now(tz=timezone.utc))
+14 −0
Original line number Diff line number Diff line
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Loading