AnalyticsBackendService.py

# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import json
import logging
import threading
from common.tools.service.GenericGrpcService import GenericGrpcService
from analytics.backend.service.SparkStreaming import SparkStreamer
from common.tools.kafka.Variables import KafkaConfig, KafkaTopic
from confluent_kafka import Consumer as KafkaConsumer
from confluent_kafka import KafkaError

LOGGER = logging.getLogger(__name__)

class AnalyticsBackendService(GenericGrpcService):
    """
    Class listens for ...
    """
    def __init__(self, cls_name : str = __name__) -> None:
        self.running_threads = {}       # To keep track of all running analyzers 
        self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(),
                                            'group.id'           : 'analytics-frontend',
                                            'auto.offset.reset'  : 'latest'})

    def RunSparkStreamer(self, analyzer_id, analyzer):
        kpi_list      = analyzer['input_kpis'] 
        oper_list     = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())]  # TODO: update this line...
        thresholds    = analyzer['thresholds']
        window_size   = analyzer['window_size']
        window_slider = analyzer['window_slider']
        print ("Received parameters: {:} - {:} - {:} - {:} - {:}".format(
            kpi_list, oper_list, thresholds, window_size, window_slider))
        LOGGER.debug ("Received parameters: {:} - {:} - {:} - {:} - {:}".format(
            kpi_list, oper_list, thresholds, window_size, window_slider))
        try:
            stop_event = threading.Event()
            thread = threading.Thread(target=SparkStreamer, 
                            args=(kpi_list, oper_list, thresholds, window_size, window_slider, None, 
                                stop_event))
            self.running_threads[analyzer_id] = (thread, stop_event)
            # thread.start()
            LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_id))
            return True
        except Exception as e:
            print       ("Failed to initiate Analyzer backend: {:}".format(e))
            LOGGER.error("Failed to initiate Analyzer backend: {:}".format(e))
            return False

    def RunRequestListener(self)->bool:
        threading.Thread(target=self.RequestListener).start()
        return True

    def RequestListener(self):
        """
        listener for requests on Kafka topic.
        """
        consumer = self.kafka_consumer
        consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value])
        while True:
            receive_msg = consumer.poll(2.0)
            if receive_msg is None:
                continue
            elif receive_msg.error():
                if receive_msg.error().code() == KafkaError._PARTITION_EOF:
                    continue
                else:
                    print("Consumer error: {}".format(receive_msg.error()))
                    break
            analyzer    = json.loads(receive_msg.value().decode('utf-8'))
            analyzer_uuid = receive_msg.key().decode('utf-8')
            LOGGER.debug('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer))
            print       ('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer))

            if analyzer["algo_name"] is None and analyzer["oper_mode"] is None:
                self.TerminateAnalyzerBackend(analyzer_uuid)
            else:
                self.RunSparkStreamer(analyzer_uuid, analyzer)

    def TerminateAnalyzerBackend(self, analyzer_uuid):
        if analyzer_uuid in self.running_threads:
            try:
                thread, stop_event = self.running_threads[analyzer_uuid]
                stop_event.set()
                thread.join()
                del self.running_threads[analyzer_uuid]
                print      ("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid))
                LOGGER.info("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid))
                return True
            except Exception as e:
                LOGGER.error("Failed to terminate. Analyzer Id: {:} - ERROR: ".format(analyzer_uuid, e))
                return False
        else:
            print         ("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid))
            # LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid))           
            # generate confirmation towards frontend