AnalyticsBackendService.py

# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import json
import logging
import threading
from common.tools.service.GenericGrpcService import GenericGrpcService
from analytics.backend.service.SparkStreaming import SparkStreamer
from common.tools.kafka.Variables import KafkaConfig, KafkaTopic
from confluent_kafka import Consumer as KafkaConsumer
from confluent_kafka import KafkaError
from common.Constants import ServiceNameEnum
from common.Settings import get_service_port_grpc


LOGGER = logging.getLogger(__name__)

class AnalyticsBackendService(GenericGrpcService):
    """
    Class listens for ...
    """
    def __init__(self, cls_name : str = __name__) -> None:
        LOGGER.info('Init AnalyticsBackendService')
        port = get_service_port_grpc(ServiceNameEnum.ANALYTICSBACKEND)
        super().__init__(port, cls_name=cls_name)
        self.running_threads = {}       # To keep track of all running analyzers 
        self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(),
                                            'group.id'           : 'analytics-frontend',
                                            'auto.offset.reset'  : 'latest'})

    def StartSparkStreamer(self, analyzer_uuid, analyzer):
        kpi_list      = analyzer['input_kpis'] 
        oper_list     = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())]  # TODO: update this line...
        thresholds    = analyzer['thresholds']
        window_size   = analyzer['window_size']
        window_slider = analyzer['window_slider']
        print ("Received parameters: {:} - {:} - {:} - {:} - {:}".format(
            kpi_list, oper_list, thresholds, window_size, window_slider))
        LOGGER.debug ("Received parameters: {:} - {:} - {:} - {:} - {:}".format(
            kpi_list, oper_list, thresholds, window_size, window_slider))
        try:
            stop_event = threading.Event()
            thread = threading.Thread(target=SparkStreamer, 
                            args=(analyzer_uuid, kpi_list, oper_list, thresholds, stop_event,
                                  window_size, window_slider, None ))
            self.running_threads[analyzer_uuid] = (thread, stop_event)
            thread.start()
            print      ("Initiated Analyzer backend: {:}".format(analyzer_uuid))
            LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_uuid))
            return True
        except Exception as e:
            print       ("Failed to initiate Analyzer backend: {:}".format(e))
            LOGGER.error("Failed to initiate Analyzer backend: {:}".format(e))
            return False

    def StopRequestListener(self, threadInfo: tuple):
        try:
            thread, stop_event = threadInfo
            stop_event.set()
            thread.join()
            print      ("Terminating Analytics backend RequestListener")
            LOGGER.info("Terminating Analytics backend RequestListener")
            return True
        except Exception as e:
            print       ("Failed to terminate analytics backend {:}".format(e))
            LOGGER.error("Failed to terminate analytics backend {:}".format(e))
            return False

    def install_services(self):
        stop_event = threading.Event()
        thread = threading.Thread(target=self.RequestListener,
                                  args=(stop_event,) )
        thread.start()
        return (thread, stop_event)

    def RequestListener(self, stop_event):
        """
        listener for requests on Kafka topic.
        """
        consumer = self.kafka_consumer
        consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value])
        while not stop_event.is_set():
            receive_msg = consumer.poll(2.0)
            if receive_msg is None:
                continue
            elif receive_msg.error():
                if receive_msg.error().code() == KafkaError._PARTITION_EOF:
                    continue
                else:
                    print("Consumer error: {}".format(receive_msg.error()))
                    break
            analyzer    = json.loads(receive_msg.value().decode('utf-8'))
            analyzer_uuid = receive_msg.key().decode('utf-8')
            LOGGER.debug('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer))
            print       ('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer))

            if analyzer["algo_name"] is None and analyzer["oper_mode"] is None:
                self.TerminateAnalyzerBackend(analyzer_uuid)
            else:
                self.StartSparkStreamer(analyzer_uuid, analyzer)
        LOGGER.debug("Stop Event activated. Terminating...")
        print       ("Stop Event activated. Terminating...")

    def TerminateAnalyzerBackend(self, analyzer_uuid):
        if analyzer_uuid in self.running_threads:
            try:
                thread, stop_event = self.running_threads[analyzer_uuid]
                stop_event.set()
                thread.join()
                del self.running_threads[analyzer_uuid]
                print      ("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid))
                LOGGER.info("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid))
                return True
            except Exception as e:
                LOGGER.error("Failed to terminate. Analyzer Id: {:} - ERROR: {:}".format(analyzer_uuid, e))
                return False
        else:
            print         ("Analyzer not found in active collectors. Analyzer Id: {:}".format(analyzer_uuid))
            LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid))           
            # generate confirmation towards frontend