Commit 8ac130c1 authored by Waleed Akbar's avatar Waleed Akbar
Browse files

Changes in Analytics 

Proto:
- Added `Duration_s` field to the proto file.

Frontend:
- Added `SelectAnalyzer` logic.
- Improved message formatting in the `create_analyzer_filter()` function.
- Added a test case: `test_SelectAnalytics`.

Backend:
- Renamed the `RunSparkStreamer` method to `StartSparkStreamer`.
- Updated the `StartRequestListener` method to return `(thread, stop_event)`.
- Added a `StopRequestListener` method to stop the listener.

Database:
- Added the `select_with_filter` method with actual logic implementation.
- Updated the `ConvertRowToAnalyzer` method to correctly read the `operation_mode` ENUM value.
parent 54e0014b

proto/analytics_frontend.proto

+10 −8
Original line number Diff line number Diff line
@@ -35,18 +35,20 @@ enum AnalyzerOperationMode { 
  ANALYZEROPERATIONMODE_STREAMING   = 2;

}



// duration field may be added in analyzer... 

message Analyzer {

  AnalyzerId                 analyzer_id          = 1;

  string                     algorithm_name       = 2;  // The algorithm to be executed

  repeated kpi_manager.KpiId input_kpi_ids        = 3;  // The KPI Ids to be processed by the analyzer

  repeated kpi_manager.KpiId output_kpi_ids       = 4;  // The KPI Ids produced by the analyzer

  AnalyzerOperationMode      operation_mode       = 5;  // Operation mode of the analyzer

  map<string, string>        parameters           = 6; 

  float                      duration_s           = 3;  // Termiate the data analytics thread after duration (seconds); 0 = infinity time

  repeated kpi_manager.KpiId input_kpi_ids        = 4;  // The KPI Ids to be processed by the analyzer

  repeated kpi_manager.KpiId output_kpi_ids       = 5;  // The KPI Ids produced by the analyzer

  AnalyzerOperationMode      operation_mode       = 6;  // Operation mode of the analyzer

  map<string, string>        parameters           = 7;  // Add dictionary of (key, value) pairs such as (window_size, 10) etc.

  // In batch mode... 

  float                      batch_min_duration_s = 7;  // ..., min duration to collect before executing batch

  float                      batch_max_duration_s = 8;  // ..., max duration collected to execute the batch

  uint64                     batch_min_size       = 9;  // ..., min number of samples to collect before executing batch

  uint64                     batch_max_size       = 10; // ..., max number of samples collected to execute the batch

  float                      batch_min_duration_s = 8;  // ..., min duration to collect before executing batch

  float                      batch_max_duration_s = 9;  // ..., max duration collected to execute the batch

  uint64                     batch_min_size       = 10; // ..., min number of samples to collect before executing batch

  uint64                     batch_max_size       = 11; // ..., max number of samples collected to execute the batch

}



message AnalyzerFilter {

src/analytics/backend/service/AnalyticsBackendService.py

+26 −8
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ class AnalyticsBackendService(GenericGrpcService): 
                                            'group.id'           : 'analytics-frontend',

                                            'auto.offset.reset'  : 'latest'})



    def RunSparkStreamer(self, analyzer_id, analyzer):

    def StartSparkStreamer(self, analyzer_id, analyzer):

        kpi_list      = analyzer['input_kpis'] 

        oper_list     = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())]  # TODO: update this line...

        thresholds    = analyzer['thresholds']
@@ -59,17 +59,33 @@ class AnalyticsBackendService(GenericGrpcService): 
            LOGGER.error("Failed to initiate Analyzer backend: {:}".format(e))

            return False



    def RunRequestListener(self)->bool:

        threading.Thread(target=self.RequestListener).start()

    def StopRequestListener(self, threadInfo: tuple):

        try:

            thread, stop_event = threadInfo

            stop_event.set()

            thread.join()

            print      ("Terminating Analytics backend RequestListener")

            LOGGER.info("Terminating Analytics backend RequestListener")

            return True

        except Exception as e:

            print       ("Failed to terminate analytics backend {:}".format(e))

            LOGGER.error("Failed to terminate analytics backend {:}".format(e))

            return False



    def StartRequestListener(self)->tuple:

        stop_event = threading.Event()

        thread = threading.Thread(target=self.RequestListener,

                                  args=(stop_event,) )

        thread.start()

        return (thread, stop_event)



    def RequestListener(self):

    def RequestListener(self, stop_event):

        """

        listener for requests on Kafka topic.

        """

        consumer = self.kafka_consumer

        consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value])

        while True:

        while not stop_event.is_set():

            receive_msg = consumer.poll(2.0)

            if receive_msg is None:

                continue
@@ -87,7 +103,9 @@ class AnalyticsBackendService(GenericGrpcService): 
            if analyzer["algo_name"] is None and analyzer["oper_mode"] is None:

                self.TerminateAnalyzerBackend(analyzer_uuid)

            else:

                self.RunSparkStreamer(analyzer_uuid, analyzer)

                self.StartSparkStreamer(analyzer_uuid, analyzer)

        LOGGER.debug("Stop Event activated. Terminating...")

        print       ("Stop Event activated. Terminating...")



    def TerminateAnalyzerBackend(self, analyzer_uuid):

        if analyzer_uuid in self.running_threads:
@@ -104,5 +122,5 @@ class AnalyticsBackendService(GenericGrpcService): 
                return False

        else:

            print         ("Analyzer not found in active collectors. Analyzer Id: {:}".format(analyzer_uuid))

            # LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid))           

            LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid))           

            # generate confirmation towards frontend

src/analytics/backend/service/SparkStreaming.py

+2 −1
Original line number Diff line number Diff line
@@ -142,6 +142,7 @@ def SparkStreamer(kpi_list, oper_list, thresholds, stop_event, 
        # Loop to check for stop event flag. To be set by stop collector method.

        while True:

            if stop_event.is_set():

                LOGGER.debug("Stop Event activated. Terminating in 5 seconds...")

                print       ("Stop Event activated. Terminating in 5 seconds...")

                time.sleep(5)

                queryHandler.stop()

src/analytics/backend/tests/test_backend.py

+18 −3
Original line number Diff line number Diff line
@@ -12,7 +12,9 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import time

import logging

import threading

from common.tools.kafka.Variables import KafkaTopic

from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService

from analytics.backend.tests.messages import get_kpi_id_list, get_operation_list, get_threshold_dict
@@ -30,12 +32,25 @@ def test_validate_kafka_topics(): 
    response = KafkaTopic.create_all_topics()

    assert isinstance(response, bool)



def test_RunRequestListener():

def test_StartRequestListener():

    LOGGER.info('test_RunRequestListener')

    AnalyticsBackendServiceObj = AnalyticsBackendService()

    response = AnalyticsBackendServiceObj.RunRequestListener()

    response = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event)

    LOGGER.debug(str(response)) 

    assert isinstance(response, tuple)



def test_StopRequestListener():

    LOGGER.info('test_RunRequestListener')

    LOGGER.info('Initiating StartRequestListener...')

    AnalyticsBackendServiceObj = AnalyticsBackendService()

    response_thread = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event)

    # LOGGER.debug(str(response_thread))

    time.sleep(10)

    LOGGER.info('Initiating StopRequestListener...')

    AnalyticsBackendServiceObj = AnalyticsBackendService()

    response = AnalyticsBackendServiceObj.StopRequestListener(response_thread)

    LOGGER.debug(str(response)) 

    assert isinstance(response, bool)



def test_SparkListener():

    LOGGER.info('test_RunRequestListener')

src/analytics/database/AnalyzerModel.py

+1 −1
Original line number Diff line number Diff line
@@ -87,7 +87,7 @@ class Analyzer(Base): 
        response                              = analytics_frontend_pb2.Analyzer()

        response.analyzer_id.analyzer_id.uuid = row.analyzer_id

        response.algorithm_name               = row.algorithm_name

        response.operation_mode               = row.operation_mode

        response.operation_mode               = row.operation_mode.value

        response.parameters.update(row.parameters)

        

        for input_kpi_id in row.input_kpi_ids: