Commit 5883eac9 authored by Waleed Akbar's avatar Waleed Akbar
Browse files

KPI Value writer and API in progress

parent b3f13a9f

src/common/tools/kafka/Variables.py

+46 −1
Original line number Diff line number Diff line
@@ -12,10 +12,17 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import logging

from enum import Enum

from confluent_kafka import KafkaException

from confluent_kafka.admin import AdminClient, NewTopic





LOGGER = logging.getLogger(__name__)



class KafkaConfig(Enum):

    SERVER_IP    = "127.0.0.1:9092"

    ADMIN_CLIENT =  AdminClient({'bootstrap.servers': SERVER_IP})



class KafkaTopic(Enum):

    REQUEST  = 'topic_request'
@@ -24,4 +31,42 @@ class KafkaTopic(Enum): 
    LABELED  = 'topic_labeled'

    VALUE    = 'topic_value'



    @staticmethod

    def create_all_topics() -> bool:

        """

            Method to create Kafka topics defined as class members

        """

        # LOGGER.debug("Topics to be created: {:}".format(KafkaTopic.__members__.values()))

        # LOGGER.debug("Topics to be created: {:}".format(KafkaTopic.__members__.keys()))

        # LOGGER.debug("Topics to be created: {:}".format([member.value for member in KafkaTopic]))

        all_topics = [member.value for member in KafkaTopic]

        if( KafkaTopic.create_new_topic_if_not_exists( all_topics )):

            LOGGER.debug("All topics created sucsessfully")

            return True

        else:

            LOGGER.debug("Error creating all topics")

            return False

    

    @staticmethod

    def create_new_topic_if_not_exists(new_topics: list) -> bool:

        """

        Method to create Kafka topic if it does not exist.

        Args:

            list of topic: containing the topic name(s) to be created on Kafka

        """

        LOGGER.debug("Recevied topic List: {:}".format(new_topics))

        for topic in new_topics:

            try:

                topic_metadata = KafkaConfig.ADMIN_CLIENT.value.list_topics(timeout=5)

                if topic not in topic_metadata.topics:

                    # If the topic does not exist, create a new topic

                    print(f"Topic '{topic}' does not exist. Creating...")

                    LOGGER.debug("Topic {:} does not exist. Creating...".format(topic))

                    new_topic = NewTopic(topic, num_partitions=1, replication_factor=1)

                    KafkaConfig.ADMIN_CLIENT.value.create_topics([new_topic])

            except Exception as e:

                LOGGER.debug("Failed to create topic: {:}".format(e))

                return False

        return True



# create all topics after the deployments (Telemetry and Analytics)
 
 No newline at end of file

src/kpi_manager/service/__main__.py

+1 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name 
    terminate.set()



def start_kpi_manager(name_mapping : NameMapping):

    LOGGER.info('Start Monitoring...',)

    LOGGER.info('Start Kpi Manager...',)



    events_collector = EventsDeviceCollector(name_mapping)

    events_collector.start()

src/kpi_manager/tests/test_kpi_manager.py

+14 −14
Original line number Diff line number Diff line
@@ -211,14 +211,14 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab 
#     LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response))

#     assert isinstance(del_response, Empty)



# def test_GetKpiDescriptor(kpi_manager_client):

#     LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ")

#     # adding KPI

#     response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request())

#     # get KPI

#     response = kpi_manager_client.GetKpiDescriptor(response_id)

#     LOGGER.info("Response gRPC message object: {:}".format(response))

#     assert isinstance(response, KpiDescriptor)

def test_GetKpiDescriptor(kpi_manager_client):

    LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ")

    # adding KPI

    response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request())

    # get KPI

    response = kpi_manager_client.GetKpiDescriptor(response_id)

    LOGGER.info("Response gRPC message object: {:}".format(response))

    assert isinstance(response, KpiDescriptor)



# def test_SelectKpiDescriptor(kpi_manager_client):

#     LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ")
@@ -229,12 +229,12 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab 
#     LOGGER.info("Response gRPC message object: {:}".format(response))

#     assert isinstance(response, KpiDescriptorList)



def test_set_list_of_KPIs(kpi_manager_client):

    LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ")

    KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"]

    # adding KPI

    for kpi in KPIs_TO_SEARCH:

       kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi))

# def test_set_list_of_KPIs(kpi_manager_client):

#     LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ")

#     KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"]

#     # adding KPI

#     for kpi in KPIs_TO_SEARCH:

#        kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi))

    



# ---------- 2nd Iteration Tests -----------------

src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py

+19 −11
Original line number Diff line number Diff line
@@ -12,8 +12,8 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import logging, grpc

from typing import Tuple, Any

import logging, grpc, json

from typing import Tuple, Any, List, Dict

from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method

from common.tools.kafka.Variables import KafkaConfig, KafkaTopic
@@ -37,22 +37,30 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): 
    def StoreKpiValues(self, request: KpiValueList, grpc_context: grpc.ServicerContext

                       ) -> Empty:

        LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request))

        producer_obj = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_IP.value})

        

        producer_obj = KafkaProducer({

            'bootstrap.servers' : KafkaConfig.SERVER_IP.value    

        })

        

        for kpi_value in request.kpi_value_list:

            kpi_value_to_produce : Tuple [str, Any, Any] = (

                kpi_value.kpi_id.kpi_id,            # kpi_value.kpi_id.kpi_id.uuid

                kpi_value.timestamp,                # kpi_value.timestamp.timestamp

                kpi_value.kpi_value_type            # kpi_value.kpi_value_type.(many options)

                kpi_value.kpi_value_type            # kpi_value.kpi_value_type.(many options) how?

            )

            LOGGER.debug('KPI to produce is {:}'.format(kpi_value_to_produce))

            msg_key = "gRPC-KpiValueApi"        # str(__class__.__name__)

            # write this KPI to Kafka

            producer_obj.produce(KafkaTopic.VALUE.value, 

        

            producer_obj.produce(

                KafkaTopic.VALUE.value, 

                key   = msg_key,

                                 value = str(kpi_value_to_produce),

                # value = json.dumps(kpi_value_to_produce),

                value = kpi_value.SerializeToString(),

                callback = self.delivery_callback

            )

            producer_obj.flush()



        return Empty()



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
@@ -61,5 +69,5 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): 
        LOGGER.debug('SelectKpiValues: Received gRPC message object: {:}'.format(request))



    def delivery_callback(self, err, msg):

        if err: print(f'Message delivery failed: {err}')

        else:   print(f'Message delivered to topic {msg.topic()}')
 
        if err: LOGGER.debug('Message delivery failed: {:}'.format(err))

        else:   print('Message delivered to topic {:}'.format(msg.topic()))

src/kpi_value_api/tests/test_kpi_value_api.py

+9 −0
Original line number Diff line number Diff line
@@ -17,14 +17,17 @@ import os, logging, pytest 


from common.proto.context_pb2 import Empty

from common.Constants import ServiceNameEnum

from common.tools.kafka.Variables import KafkaTopic

from common.Settings import ( 

    ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc)





from kpi_value_api.service.NameMapping import NameMapping

from kpi_value_api.service.KpiValueApiService import KpiValueApiService

from kpi_value_api.client.KpiValueApiClient import KpiValueApiClient

from kpi_value_api.tests.messages import create_kpi_value_list





LOCAL_HOST = '127.0.0.1'



KPIVALUEAPI_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.KPIVALUEAPI)  # type: ignore
@@ -77,6 +80,12 @@ def kpi_value_api_client(kpi_value_api_service : KpiValueApiService ): 
# Tests Implementation of Kpi Value Api

###########################



def test_validate_kafka_topics():

    LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ")

    response = KafkaTopic.create_all_topics()

    assert isinstance(response, bool)





def test_store_kpi_values(kpi_value_api_client):

    LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ")

    response = kpi_value_api_client.StoreKpiValues(create_kpi_value_list())