Feat/monitoring major functional updates

Feat/monitoring major functional updates

Merge request reports

Loading
+29 −30
Changes for proto/monitoring.proto: 29 added lines, 30 removed lines.
Original line number Diff line number Diff line
@@ -19,24 +19,24 @@ import "context.proto";
import "kpi_sample_types.proto";

service MonitoringService {
  rpc SetKpi                (KpiDescriptor      ) returns (KpiId               ) {}
  rpc DeleteKpi             (KpiId              ) returns (context.Empty       ) {}
  rpc GetKpiDescriptor      (KpiId              ) returns (KpiDescriptor       ) {}
  rpc GetKpiDescriptorList  (context.Empty      ) returns (KpiDescriptorList   ) {}
  rpc IncludeKpi            (Kpi                ) returns (context.Empty       ) {}
  rpc MonitorKpi            (MonitorKpiRequest  ) returns (context.Empty       ) {}
  rpc QueryKpiData          (KpiQuery           ) returns (KpiList             ) {}
  rpc SetKpiSubscription    (SubsDescriptor     ) returns (stream KpiList      ) {}
  rpc GetSubsDescriptor     (SubscriptionID     ) returns (SubsDescriptor      ) {}
  rpc GetSubscriptions      (context.Empty      ) returns (SubsIDList          ) {}
  rpc DeleteSubscription    (SubscriptionID     ) returns (context.Empty       ) {}
  rpc SetKpiAlarm           (AlarmDescriptor    ) returns (AlarmID             ) {}
  rpc GetAlarms             (context.Empty      ) returns (AlarmIDList         ) {}
  rpc GetAlarmDescriptor    (AlarmID            ) returns (AlarmDescriptor     ) {}
  rpc GetAlarmResponseStream(AlarmSubscription  ) returns (stream AlarmResponse) {}
  rpc DeleteAlarm           (AlarmID            ) returns (context.Empty       ) {}
  rpc GetStreamKpi          (KpiId              ) returns (stream Kpi          ) {}
  rpc GetInstantKpi         (KpiId              ) returns (KpiList             ) {}
  rpc SetKpi                (KpiDescriptor      ) returns (KpiId               ) {} // Stable not final
  rpc DeleteKpi             (KpiId              ) returns (context.Empty       ) {} // Stable and final
  rpc GetKpiDescriptor      (KpiId              ) returns (KpiDescriptor       ) {} // Stable and final
  rpc GetKpiDescriptorList  (context.Empty      ) returns (KpiDescriptorList   ) {} // Stable and final
  rpc IncludeKpi            (Kpi                ) returns (context.Empty       ) {} // Stable and final
  rpc MonitorKpi            (MonitorKpiRequest  ) returns (context.Empty       ) {} // Stable and final
  rpc QueryKpiData          (KpiQuery           ) returns (KpiList             ) {} // Not implemented
  rpc SetKpiSubscription    (SubsDescriptor     ) returns (stream SubsResponse ) {} // Stable not final
  rpc GetSubsDescriptor     (SubscriptionID     ) returns (SubsDescriptor      ) {} // Stable and final
  rpc GetSubscriptions      (context.Empty      ) returns (SubsList            ) {} // Stable and final
  rpc DeleteSubscription    (SubscriptionID     ) returns (context.Empty       ) {} // Stable and final
  rpc SetKpiAlarm           (AlarmDescriptor    ) returns (AlarmID             ) {} // Stable not final
  rpc GetAlarms             (context.Empty      ) returns (AlarmList           ) {} // Stable and final
  rpc GetAlarmDescriptor    (AlarmID            ) returns (AlarmDescriptor     ) {} // Stable and final
  rpc GetAlarmResponseStream(AlarmSubscription  ) returns (stream AlarmResponse) {} // Not Stable not final
  rpc DeleteAlarm           (AlarmID            ) returns (context.Empty       ) {} // Stable and final
  rpc GetStreamKpi          (KpiId              ) returns (stream Kpi          ) {} // Stable not final
  rpc GetInstantKpi         (KpiId              ) returns (Kpi             ) {} // Stable not final
}

message KpiDescriptor {
@@ -58,7 +58,7 @@ message MonitorKpiRequest {
}

message KpiQuery {
  repeated KpiId    kpi_id              = 1;
  KpiId    kpi_id              = 1;
  float             monitoring_window_s = 2;
  float             sampling_rate_s     = 3;
  uint32            last_n_samples      = 4;  // used when you want something like "get the last N many samples
@@ -99,7 +99,7 @@ message KpiValue {


message KpiList {
  repeated Kpi kpi_list = 1;
  repeated Kpi kpi = 1;
}

message KpiDescriptorList {
@@ -122,19 +122,19 @@ message SubscriptionID {

message SubsResponse {
  SubscriptionID   subs_id  = 1;
  repeated KpiList kpi_list = 2;
  KpiList          kpi_list = 2;
}

message SubsIDList {
  repeated SubscriptionID subs_list = 1;
message SubsList {
  repeated SubsDescriptor subs_descriptor = 1;
}

message AlarmDescriptor {
  AlarmID                     alarm_id              = 1;
  string                      alarm_description     = 2;
  string                      name                  = 3;
  repeated KpiId              kpi_id                = 4;
  repeated KpiValueRange      kpi_value_range       = 5;
  KpiId                       kpi_id                = 4;
  KpiValueRange               kpi_value_range       = 5;
  context.Timestamp           timestamp             = 6;
}

@@ -143,7 +143,7 @@ message AlarmID{
}

message AlarmSubscription{
  AlarmID alarmID                   = 1;
  AlarmID alarm_id                  = 1;
  float   subscription_timeout_s    = 2;
  float   subscription_frequency_ms = 3;
}
@@ -151,10 +151,9 @@ message AlarmSubscription{
message AlarmResponse {
  AlarmID           alarm_id  = 1;
  string            text      = 2;
  KpiValue          kpi_value = 3;
  context.Timestamp timestamp = 4;
  KpiList           kpi_list  = 3;
}

message AlarmIDList {
    repeated AlarmID alarm_list = 1;
message AlarmList {
    repeated AlarmDescriptor alarm_descriptor = 1;
}
+8 −8
Changes for src/monitoring/client/MonitoringClient.py: 8 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -21,8 +21,8 @@ from common.tools.client.RetryDecorator import retry, delay_exponential
from common.tools.grpc.Tools import grpc_message_to_json_string
from common.proto.context_pb2 import Empty
from common.proto.monitoring_pb2 import Kpi, KpiDescriptor, KpiId, MonitorKpiRequest, \
    KpiDescriptorList, KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsIDList, \
    AlarmDescriptor, AlarmID, AlarmIDList, AlarmResponse, AlarmSubscription
    KpiDescriptorList, KpiQuery, KpiList, SubsDescriptor, SubscriptionID, SubsList, \
    SubsResponse, AlarmDescriptor, AlarmID, AlarmList, AlarmResponse, AlarmSubscription
from common.proto.monitoring_pb2_grpc import MonitoringServiceStub

LOGGER = logging.getLogger(__name__)
@@ -100,10 +100,10 @@ class MonitoringClient:
        return response

    @RETRY_DECORATOR
    def SubscribeKpi(self, request : SubsDescriptor) -> Iterator[KpiList]:
        LOGGER.debug('SubscribeKpi: {:s}'.format(grpc_message_to_json_string(request)))
        response = self.stub.SubscribeKpi(request)
        LOGGER.debug('SubscribeKpi result: {:s}'.format(grpc_message_to_json_string(response)))
    def SetKpiSubscription(self, request : SubsDescriptor) -> Iterator[SubsResponse]:
        LOGGER.debug('SetKpiSubscription: {:s}'.format(grpc_message_to_json_string(request)))
        response = self.stub.SetKpiSubscription(request)
        LOGGER.debug('SetKpiSubscription result: {:s}'.format(grpc_message_to_json_string(response)))
        return response

    @RETRY_DECORATOR
@@ -114,7 +114,7 @@ class MonitoringClient:
        return response

    @RETRY_DECORATOR
    def GetSubscriptions(self, request : Empty) -> SubsIDList:
    def GetSubscriptions(self, request : Empty) -> SubsList:
        LOGGER.debug('GetSubscriptions: {:s}'.format(grpc_message_to_json_string(request)))
        response = self.stub.GetSubscriptions(request)
        LOGGER.debug('GetSubscriptions result: {:s}'.format(grpc_message_to_json_string(response)))
@@ -135,7 +135,7 @@ class MonitoringClient:
        return response

    @RETRY_DECORATOR
    def GetAlarms(self, request : Empty) -> AlarmIDList:
    def GetAlarms(self, request : Empty) -> AlarmList:
        LOGGER.debug('GetAlarms: {:s}'.format(grpc_message_to_json_string(request)))
        response = self.stub.GetAlarms(request)
        LOGGER.debug('GetAlarms result: {:s}'.format(grpc_message_to_json_string(response)))
+32 −0
Changes for src/monitoring/service/AlarmManager.py: 32 added lines, 0 removed lines.
Original line number Diff line number Diff line
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ProcessPoolExecutor
from apscheduler.jobstores.base import JobLookupError
from datetime import datetime
import time
import logging

LOGGER = logging.getLogger(__name__)

class AlarmManager():
    def __init__(self, metrics_db):
        self.metrics_db = metrics_db
        self.scheduler = BackgroundScheduler(executors={'processpool': ProcessPoolExecutor(max_workers=20)})
        self.scheduler.start()
        LOGGER.info("Alarm Manager Initialized")

    def create_alarm(self, alarm_queue,alarm_id, kpi_id, kpiMinValue, kpiMaxValue, inRange, includeMinValue, includeMaxValue, subscription_frequency_ms, subscription_timeout_s=None):
        start_date=None
        end_date=None
        if subscription_timeout_s:
            start_timestamp=time.time()
            start_date=datetime.fromtimestamp(start_timestamp)
            end_date=datetime.fromtimestamp(start_timestamp+subscription_timeout_s)
        self.scheduler.add_job(self.metrics_db.get_alarm_data, args=(alarm_queue,kpi_id, kpiMinValue, kpiMaxValue, inRange, includeMinValue, includeMaxValue, subscription_frequency_ms),trigger='interval', seconds=(subscription_frequency_ms/1000), start_date=start_date, end_date=end_date, id=alarm_id)
        LOGGER.debug(f"Alarm job {alarm_id} succesfully created")

    def delete_alarm(self, alarm_id):
        try:
            self.scheduler.remove_job(alarm_id)
            LOGGER.debug(f"Alarm job {alarm_id} succesfully deleted")
        except (Exception, JobLookupError) as e:
            LOGGER.debug(f"Alarm job {alarm_id} does not exists")
+248 −0
Changes for src/monitoring/service/ManagementDBTools.py: 248 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sqlite3
import logging

LOGGER = logging.getLogger(__name__)

class ManagementDB():
    def __init__(self, database):
        try:
            self.client = sqlite3.connect(database, check_same_thread=False)
            self.create_monitoring_table()
            self.create_subscription_table()
            self.create_alarm_table()
            LOGGER.info("ManagementDB initialized")
        except:
            LOGGER.info("ManagementDB cannot be initialized")
            raise Exception("Critical error in the monitoring component")
        
    def create_monitoring_table(self):
        try:
            result=self.client.execute("""
                CREATE TABLE IF NOT EXISTS kpi(
                    kpi_id INTEGER PRIMARY KEY AUTOINCREMENT,
                    kpi_description TEXT,
                    kpi_sample_type INTEGER,
                    device_id INTEGER,
                    endpoint_id INTEGER,
                    service_id INTEGER
                );
            """)
            LOGGER.debug("KPI table created in the ManagementDB")
        except sqlite3.Error as e:
            LOGGER.debug(f"KPI table cannot be created in the ManagementD. {e}")
            raise Exception
    
    def create_subscription_table(self):
        try:
            result= self.client.execute("""
                CREATE TABLE IF NOT EXISTS subscription(
                    subs_id INTEGER PRIMARY KEY AUTOINCREMENT,
                    kpi_id INTEGER,
                    subscriber TEXT,
                    sampling_duration_s REAL,
                    sampling_interval_s REAL,
                    start_timestamp REAL,
                    end_timestamp REAL
                );
            """)
            LOGGER.info("Subscription table created in the ManagementDB")
        except sqlite3.Error as e:
            LOGGER.debug(f"Subscription table cannot be created in the ManagementDB. {e}")
            raise Exception

    def create_alarm_table(self):
        try:
            result=self.client.execute("""
                CREATE TABLE IF NOT EXISTS alarm(
                    alarm_id INTEGER PRIMARY KEY AUTOINCREMENT,
                    alarm_description TEXT,
                    alarm_name TEXT,
                    kpi_id INTEGER,
                    kpi_min_value REAL,
                    kpi_max_value REAL,
                    in_range INTEGER,
                    include_min_value INTEGER,
                    include_max_value INTEGER
                );
            """)
            LOGGER.info("Alarm table created in the ManagementDB")
        except sqlite3.Error as e:
            LOGGER.debug(f"Alarm table cannot be created in the ManagementDB. {e}")
            raise Exception

    def insert_KPI(self,kpi_description,kpi_sample_type,device_id,endpoint_id,service_id):
        try:
            c = self.client.cursor()
            c.execute("SELECT kpi_id FROM kpi WHERE device_id is ? AND kpi_sample_type is ? AND endpoint_id is ? AND service_id is ?",(device_id,kpi_sample_type,endpoint_id,service_id))
            data=c.fetchone()
            if data is None:
                c.execute("INSERT INTO kpi (kpi_description,kpi_sample_type,device_id,endpoint_id,service_id) VALUES (?,?,?,?,?)", (kpi_description,kpi_sample_type,device_id,endpoint_id,service_id))
                self.client.commit()
                kpi_id = c.lastrowid
                LOGGER.debug(f"KPI {kpi_id} succesfully inserted in the ManagementDB")
                return kpi_id
            else:
                kpi_id = data[0]
                LOGGER.debug(f"KPI {kpi_id} already exists")
                return kpi_id
        except sqlite3.Error as e:
            LOGGER.debug("KPI cannot be inserted in the ManagementDB: {e}")
            
    def insert_subscription(self,kpi_id,subscriber,sampling_duration_s,sampling_interval_s,start_timestamp, end_timestamp):
        try:
            c = self.client.cursor()
            c.execute("SELECT subs_id FROM subscription WHERE kpi_id is ? AND subscriber is ? AND sampling_duration_s is ? AND sampling_interval_s is ? AND start_timestamp is ? AND end_timestamp is ?",(kpi_id,subscriber,sampling_duration_s,sampling_interval_s,start_timestamp, end_timestamp))
            data=c.fetchone()
            if data is None:
                c.execute("INSERT INTO subscription (kpi_id,subscriber,sampling_duration_s,sampling_interval_s,start_timestamp, end_timestamp) VALUES (?,?,?,?,?,?)", (kpi_id,subscriber,sampling_duration_s,sampling_interval_s,start_timestamp, end_timestamp))
                self.client.commit()
                subs_id = c.lastrowid
                LOGGER.debug(f"Subscription {subs_id} succesfully inserted in the ManagementDB")
                return subs_id
            else:
                subs_id = data[0]
                LOGGER.debug(f"Subscription {subs_id} already exists")
                return subs_id
        except sqlite3.Error as e:
            LOGGER.debug("Subscription cannot be inserted in the ManagementDB: {e}")

    def insert_alarm(self,alarm_description,alarm_name,kpi_id,kpi_min_value,kpi_max_value,in_range,include_min_value,include_max_value):
        try:
            c = self.client.cursor()
            c.execute("SELECT alarm_id FROM alarm WHERE alarm_description is ? AND alarm_name is ? AND kpi_id is ? AND kpi_min_value is ? AND kpi_max_value is ? AND in_range is ? AND include_min_value is ? AND include_max_value is ?",(alarm_description,alarm_name,kpi_id,kpi_min_value,kpi_max_value,in_range,include_min_value,include_max_value))
            data=c.fetchone()
            if data is None:
                c.execute("INSERT INTO alarm (alarm_description, alarm_name, kpi_id, kpi_min_value, kpi_max_value, in_range, include_min_value, include_max_value) VALUES (?,?,?,?,?,?,?,?)", (alarm_description,alarm_name,kpi_id,kpi_min_value,kpi_max_value,in_range,include_min_value,include_max_value))
                self.client.commit()
                alarm_id=c.lastrowid
                LOGGER.debug(f"Alarm {alarm_id} succesfully inserted in the ManagementDB")
                return alarm_id
            else:
                alarm_id=data[0]
                LOGGER.debug(f"Alarm {alarm_id} already exists")
                return alarm_id
        except sqlite3.Error as e:
            LOGGER.debug(f"Alarm cannot be inserted in the ManagementDB: {e}")

    def delete_KPI(self,kpi_id):
        try:
            c = self.client.cursor()
            c.execute("SELECT * FROM kpi WHERE kpi_id is ?",(kpi_id,))       
            data=c.fetchone()
            if data is None:
                LOGGER.debug(f"KPI {kpi_id} does not exists")
                return False
            else:
                c.execute("DELETE FROM kpi WHERE kpi_id is ?",(kpi_id,))
                self.client.commit()
                LOGGER.debug(f"KPI {kpi_id} deleted from the ManagementDB")
                return True
        except sqlite3.Error as e:
            LOGGER.debug(f"KPI cannot be deleted from the ManagementDB: {e}")

    def delete_subscription(self,subs_id):
        try:
            c = self.client.cursor()
            c.execute("SELECT * FROM subscription WHERE subs_id is ?",(subs_id,))       
            data=c.fetchone()
            if data is None:
                LOGGER.debug(f"Subscription {subs_id} does not exists")
                return False
            else:
                c.execute("DELETE FROM subscription WHERE subs_id is ?",(subs_id,))
                self.client.commit()
                LOGGER.debug(f"Subscription {subs_id} deleted from the ManagementDB")
                return True
        except sqlite3.Error as e:
            LOGGER.debug(f"Subscription cannot be deleted from the ManagementDB: {e}")

    def delete_alarm(self,alarm_id):
        try:
            c = self.client.cursor()
            c.execute("SELECT * FROM alarm WHERE alarm_id is ?",(alarm_id,))       
            data=c.fetchone()
            if data is None:
                LOGGER.debug(f"Alarm {alarm_id} does not exists")
                return False
            else:
                c.execute("DELETE FROM alarm WHERE alarm_id is ?",(alarm_id,))
                self.client.commit()
                LOGGER.debug(f"Alarm {alarm_id} deleted from the ManagementDB")
                return True
        except sqlite3.Error as e:
            LOGGER.debug(f"Alarm cannot be deleted from the ManagementDB: {e}")

    def get_KPI(self,kpi_id):
        try:
            data = self.client.execute("SELECT * FROM kpi WHERE kpi_id is ?",(kpi_id,)).fetchone()
            if data:
                LOGGER.debug(f"KPI {kpi_id} succesfully retrieved from the ManagementDB")
                return data
            else:
                LOGGER.debug(f"KPI {kpi_id} does not exists")
                return data
        except sqlite3.Error as e:
            LOGGER.debug(f"KPI {kpi_id} cannot be retrieved from the ManagementDB: {e}")

    def get_subscription(self,subs_id):
        try:
            data = self.client.execute("SELECT * FROM subscription WHERE subs_id is ?",(subs_id,)).fetchone()
            if data:
                LOGGER.debug(f"Subscription {subs_id} succesfully retrieved from the ManagementDB")
                return data
            else:
                LOGGER.debug(f"Subscription {subs_id} does not exists")
                return data
        except sqlite3.Error as e:
            LOGGER.debug(f"Subscription {subs_id} cannot be retrieved from the ManagementDB: {e}")

    def get_alarm(self,alarm_id):
        try:
            data = self.client.execute("SELECT * FROM alarm WHERE alarm_id is ?",(alarm_id,)).fetchone()
            if data:
                LOGGER.debug(f"Alarm {alarm_id} succesfully retrieved from the ManagementDB")
                return data
            else:
                print(data)
                LOGGER.debug(f"Alarm {alarm_id} does not exists")
                return data
        except sqlite3.Error as e:
            LOGGER.debug(f"Alarm {alarm_id} cannot be retrieved from the ManagementDB: {e}")
        
    def get_KPIS(self):
        try:
            data = self.client.execute("SELECT * FROM kpi").fetchall()
            LOGGER.debug(f"KPIs succesfully retrieved from the ManagementDB")
            return data
        except sqlite3.Error as e:
            LOGGER.debug(f"KPIs cannot be retrieved from the ManagementDB: {e}")

    def get_subscriptions(self):
        try:
            data = self.client.execute("SELECT * FROM subscription").fetchall()
            LOGGER.debug(f"Subscriptions succesfully retrieved from the ManagementDB")
            return data
        except sqlite3.Error as e:
            LOGGER.debug(f"Subscriptions cannot be retrieved from the ManagementDB: {e}")

    def get_alarms(self):
        try:
            data = self.client.execute("SELECT * FROM alarm").fetchall()
            LOGGER.debug(f"Alarms succesfully retrieved from the ManagementDB")
            return data
        except sqlite3.Error as e:
            LOGGER.debug(f"Alarms cannot be retrieved from the ManagementDB: {e}")
 No newline at end of file
+219 −22
Changes for src/monitoring/service/MetricsDBTools.py: 219 added lines, 22 removed lines.
Original line number Diff line number Diff line
@@ -12,26 +12,94 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import time
from random import random

from questdb.ingress import Sender, IngressError
import requests
import json
import logging
import datetime
from common.tools.timestamp.Converters import timestamp_float_to_string, timestamp_utcnow_to_float
import psycopg2

LOGGER = logging.getLogger(__name__)


class MetricsDB():
  def __init__(self, host, ilp_port, rest_port, table):
    def __init__(self, host, ilp_port=9009, rest_port=9000, table="monitoring", commit_lag_ms=1000, retries=10,
                 postgre=False, postgre_port=8812, postgre_user='admin', postgre_password='quest'):
        try:
            self.host = host
            self.ilp_port = int(ilp_port)
            self.rest_port = rest_port
            self.table = table
            self.commit_lag_ms = commit_lag_ms
            self.retries = retries
            self.postgre = postgre
            self.postgre_port = postgre_port
            self.postgre_user = postgre_user
            self.postgre_password = postgre_password
            self.create_table()
            LOGGER.info("MetricsDB initialized")
        except:
            LOGGER.info("MetricsDB cannot be initialized")
            raise Exception("Critical error in the monitoring component")

    def is_postgre_enabled(self):
        LOGGER.info(f"PostgreSQL is {self.postgre}")
        return self.postgre

    def get_retry_number(self):
        LOGGER.info(f"Retry number is {self.retries}")
        return self.retries

    def get_commit_lag(self):
        LOGGER.info(f"Commit lag of monitoring queries is {self.commit_lag_ms} ms")
        return self.commit_lag_ms

    def enable_postgre_mode(self):
        self.postgre = True
        LOGGER.info("MetricsDB PostgreSQL query mode enabled")

    def disable_postgre_mode(self):
        self.postgre = False
        LOGGER.info("MetricsDB REST query mode enabled")

    def set_postgre_credentials(self, user, password):
        self.postgre_user = user
        self.postgre_password = password
        LOGGER.info("MetricsDB PostgreSQL credentials changed")

    def set_retry_number(self, retries):
        self.retries = retries
        LOGGER.info(f"Retriy number changed to {retries}")

    def set_commit_lag(self, commit_lag_ms):
        self.commit_lag_ms = commit_lag_ms
        LOGGER.info(f"Commit lag of monitoring queries changed to {commit_lag_ms} ms")

    def create_table(self):
        try:
            query = f'CREATE TABLE IF NOT EXISTS {self.table}' \
                    '(kpi_id SYMBOL,' \
                    'kpi_sample_type SYMBOL,' \
                    'device_id SYMBOL,' \
                    'endpoint_id SYMBOL,' \
                    'service_id SYMBOL,' \
                    'timestamp TIMESTAMP,' \
                    'kpi_value DOUBLE)' \
                    'TIMESTAMP(timestamp);'
            result = self.run_query(query)
            if (result == True):
                LOGGER.info(f"Table {self.table} created")
        except (Exception) as e:
            LOGGER.debug(f"Table {self.table} cannot be created. {e}")
            raise Exception

    def write_KPI(self, time, kpi_id, kpi_sample_type, device_id, endpoint_id, service_id, kpi_value):
        counter = 0
    number_of_retries=10
    while (counter<number_of_retries):
        while (counter < self.retries):
            try:
                with Sender(self.host, self.ilp_port) as sender:
                    sender.row(
@@ -46,30 +114,159 @@ class MetricsDB():
                            'kpi_value': kpi_value},
                        at=datetime.datetime.fromtimestamp(time))
                    sender.flush()
        counter=number_of_retries
        LOGGER.info(f"KPI written")
      except IngressError as ierr:
        # LOGGER.info(ierr)
        # LOGGER.info(f"Ingress Retry number {counter}")
                counter = self.retries
                LOGGER.debug(f"KPI written in the MetricsDB")
            except (Exception, IngressError) as e:
                counter = counter + 1

                if counter == self.retries:
                    raise Exception(f"Maximum number of retries achieved: {self.retries}")

    def run_query(self, sql_query):
        counter = 0
        while (counter < self.retries):
            try:
                query_params = {'query': sql_query, 'fmt': 'json'}
                url = f"http://{self.host}:{self.rest_port}/exec"
                response = requests.get(url, params=query_params)
                json_response = json.loads(response.text)
    LOGGER.info(f"Query executed, result:{json_response}")
                if ('ddl' in json_response):
                    LOGGER.debug(f"REST query executed succesfully, result: {json_response['ddl']}")
                    counter = self.retries
                    return True
                elif ('dataset' in json_response):
                    LOGGER.debug(f"REST query executed, result: {json_response['dataset']}")
                    counter = self.retries
                    return json_response['dataset']
            except (Exception, requests.exceptions.RequestException) as e:
                counter = counter + 1
                if counter == self.retries:
                    raise Exception(f"Maximum number of retries achieved: {self.retries}")

  def create_table(self):
    query = f'CREATE TABLE IF NOT EXISTS {self.table}'\
    '(kpi_id SYMBOL,'\
    'kpi_sample_type SYMBOL,'\
    'device_id SYMBOL,'\
    'endpoint_id SYMBOL,'\
    'service_id SYMBOL,'\
    'timestamp TIMESTAMP,'\
    'kpi_value DOUBLE)'\
    'TIMESTAMP(timestamp);'
    self.run_query(query)
    LOGGER.info(f"Table {self.table} created")
    def run_query_postgre(self, postgre_sql_query):
        connection = None
        cursor = None
        counter = 0
        while (counter < self.retries):
            try:
                connection = psycopg2.connect(
                    user=self.postgre_user,
                    password=self.postgre_password,
                    host=self.host,
                    port=self.postgre_port,
                    database=self.table)
                cursor = connection.cursor()
                cursor.execute(postgre_sql_query)
                result = cursor.fetchall()
                LOGGER.debug(f"PostgreSQL query executed, result: {result}")
                counter = self.retries
                return result
            except (Exception, psycopg2.Error) as e:
                counter = counter + 1
                if counter == self.retries:
                    raise Exception(f"Maximum number of retries achieved: {self.retries}")
            finally:
                if cursor:
                    cursor.close()
                if connection:
                    connection.close()

    def get_subscription_data(self,subs_queue, kpi_id, sampling_interval_s=1):
        try:
            end_date = timestamp_utcnow_to_float() - self.commit_lag_ms / 1000
            start_date = end_date - sampling_interval_s
            query = f"SELECT kpi_id, timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"
            if self.postgre:
                kpi_list = self.run_query_postgre(query)
                LOGGER.debug(f"kpi_list postgre: {kpi_list}")
            else:
                kpi_list = self.run_query(query)
                LOGGER.debug(f"kpi_list influx: {kpi_list}")
            if kpi_list:
                subs_queue.put_nowait(kpi_list)
                LOGGER.debug(f"New data received for subscription to KPI {kpi_id}")
            else:
                LOGGER.debug(f"No new data for the subscription to KPI {kpi_id}")
        except (Exception) as e:
            LOGGER.debug(f"Subscription data cannot be retrieved. {e}")

    def get_alarm_data(self, alarm_queue, kpi_id, kpiMinValue, kpiMaxValue, inRange=True, includeMinValue=True, includeMaxValue=True,
                       subscription_frequency_ms=1000):
        try:
            end_date = timestamp_utcnow_to_float() - self.commit_lag_ms / 1000
            start_date = end_date - subscription_frequency_ms / 1000
            query = f"SELECT kpi_id, timestamp, kpi_value FROM {self.table} WHERE kpi_id = '{kpi_id}' AND (timestamp BETWEEN '{timestamp_float_to_string(start_date)}' AND '{timestamp_float_to_string(end_date)}')"
            if self.postgre:
                kpi_list = self.run_query_postgre(query)
            else:
                kpi_list = self.run_query(query)
            if kpi_list:
                LOGGER.debug(f"New data received for alarm of KPI {kpi_id}")
                for kpi in kpi_list:
                    alarm = False
                    kpi_value = kpi[2]
                    if (kpiMinValue == kpi_value and kpiMaxValue == kpi_value and inRange):
                        alarm = True
                    elif (
                            inRange and kpiMinValue is not None and kpiMaxValue is not None and includeMinValue and includeMaxValue):
                        if (kpi_value >= kpiMinValue and kpi_value <= kpiMaxValue):
                            alarm = True
                    elif (
                            inRange and kpiMinValue is not None and kpiMaxValue is not None and includeMinValue and not includeMaxValue):
                        if (kpi_value >= kpiMinValue and kpi_value < kpiMaxValue):
                            alarm = True
                    elif (
                            inRange and kpiMinValue is not None and kpiMaxValue is not None and not includeMinValue and includeMaxValue):
                        if (kpi_value > kpiMinValue and kpi_value <= kpiMaxValue):
                            alarm = True
                    elif (
                            inRange and kpiMinValue is not None and kpiMaxValue is not None and not includeMinValue and not includeMaxValue):
                        if (kpi_value > kpiMinValue and kpi_value < kpiMaxValue):
                            alarm = True
                    elif (
                            not inRange and kpiMinValue is not None and kpiMaxValue is not None and includeMinValue and includeMaxValue):
                        if (kpi_value <= kpiMinValue or kpi_value >= kpiMaxValue):
                            alarm = True
                    elif (
                            not inRange and kpiMinValue is not None and kpiMaxValue is not None and includeMinValue and not includeMaxValue):
                        if (kpi_value <= kpiMinValue or kpi_value > kpiMaxValue):
                            alarm = True
                    elif (
                            not inRange and kpiMinValue is not None and kpiMaxValue is not None and not includeMinValue and includeMaxValue):
                        if (kpi_value < kpiMinValue or kpi_value >= kpiMaxValue):
                            alarm = True
                    elif (
                            not inRange and kpiMinValue is not None and kpiMaxValue is not None and not includeMinValue and not includeMaxValue):
                        if (kpi_value < kpiMinValue or kpi_value > kpiMaxValue):
                            alarm = True
                    elif (inRange and kpiMinValue is not None and kpiMaxValue is None and includeMinValue):
                        if (kpi_value >= kpiMinValue):
                            alarm = True
                    elif (inRange and kpiMinValue is not None and kpiMaxValue is None and not includeMinValue):
                        if (kpi_value > kpiMinValue):
                            alarm = True
                    elif (not inRange and kpiMinValue is not None and kpiMaxValue is None and not includeMinValue):
                        if (kpi_value <= kpiMinValue):
                            alarm = True
                    elif (not inRange and kpiMinValue is not None and kpiMaxValue is None and not includeMinValue):
                        if (kpi_value <= kpiMinValue):
                            alarm = True
                    elif (inRange and kpiMinValue is None and kpiMaxValue is not None and includeMaxValue):
                        if (kpi_value <= kpiMaxValue):
                            alarm = True
                    elif (inRange and kpiMinValue is None and kpiMaxValue is not None and not includeMaxValue):
                        if (kpi_value < kpiMaxValue):
                            alarm = True
                    elif (not inRange and kpiMinValue is None and kpiMaxValue is not None and not includeMaxValue):
                        if (kpi_value >= kpiMaxValue):
                            alarm = True
                    elif (not inRange and kpiMinValue is None and kpiMaxValue is not None and not includeMaxValue):
                        if (kpi_value >= kpiMaxValue):
                            alarm = True
                    if alarm:
                        # queue.append[kpi]
                        alarm_queue.put_nowait(kpi)
                        LOGGER.debug(f"Alarm of KPI {kpi_id} triggered -> kpi_value:{kpi[2]}, timestamp:{kpi[1]}")
                else:
                    LOGGER.debug(f"No new data for the alarm of KPI {kpi_id}")
        except (Exception) as e:
            LOGGER.debug(f"Alarm data cannot be retrieved. {e}")
 No newline at end of file
Loading
Loading