Resolve: "Unable to correctly extract the aggregation function names from the dictionary received as parameters in the Analyzer message"

Closes #194 (closed)

Merge request reports

Loading
+1 −1

File changed.

Contains only whitespace changes.

+8 −0
Changes for proto/kpi_value_api.proto: 8 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ import "kpi_manager.proto";
service KpiValueAPIService {
	rpc StoreKpiValues  (KpiValueList)      returns (context.Empty)    {}
	rpc SelectKpiValues (KpiValueFilter)    returns (KpiValueList)     {}
  rpc GetKpiAlarms    (kpi_manager.KpiId) returns (stream KpiAlarms) {}
}

message KpiValue {
@@ -50,3 +51,10 @@ message KpiValueFilter {
	repeated context.Timestamp start_timestamp = 2;
	repeated context.Timestamp end_timestamp   = 3;
}

message KpiAlarms {
  context.Timestamp start_timestamp = 1;
  context.Timestamp end_timestamp   = 2;
  kpi_manager.KpiId kpi_id          = 3;
  map<string, bool> alarms          = 4;
}
+6 −9
Changes for src/analytics/backend/service/AnalyticsBackendService.py: 6 added lines, 9 removed lines.
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ class AnalyticsBackendService(GenericGrpcService):

    def StartSparkStreamer(self, analyzer_uuid, analyzer):
        kpi_list      = analyzer['input_kpis'] 
        oper_list     = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())]  # TODO: update this line...
        oper_list     = [s.split('_', 1)[0] for s in list(analyzer["thresholds"].keys())]  # TODO: update this line...
        thresholds    = analyzer['thresholds']
        window_size   = analyzer['window_size']
        window_slider = analyzer['window_slider']
@@ -78,20 +78,17 @@ class AnalyticsBackendService(GenericGrpcService):
            LOGGER.error("Failed to terminate analytics backend {:}".format(e))
            return False

    def install_services(self):
        stop_event = threading.Event()
        thread = threading.Thread(target=self.RequestListener,
                                  args=(stop_event,) )
        thread.start()
        return (thread, stop_event)
    def install_servicers(self):
        threading.Thread(target=self.RequestListener, args=()).start()

    def RequestListener(self, stop_event):
    def RequestListener(self):
        """
        listener for requests on Kafka topic.
        """
        LOGGER.info("Request Listener is initiated ...")
        consumer = self.kafka_consumer
        consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value])
        while not stop_event.is_set():
        while True:
            receive_msg = consumer.poll(2.0)
            if receive_msg is None:
                continue
+4 −4
Changes for src/analytics/backend/service/SparkStreaming.py: 4 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ def SettingKafkaConsumerParams(): # TODO: create get_kafka_consumer() in comm
    return {
            # "kafka.bootstrap.servers": '127.0.0.1:9092',
            "kafka.bootstrap.servers": KafkaConfig.get_kafka_address(),
            "subscribe"              : KafkaTopic.VALUE.value,
            "subscribe"              : KafkaTopic.VALUE.value,         # topic should have atleast one message before spark session 
            "startingOffsets"        : 'latest',
            "failOnDataLoss"         : 'false'              # Optional: Set to "true" to fail the query on data loss
        }
@@ -64,7 +64,7 @@ def ApplyThresholds(aggregated_df, thresholds):
    for col_name, (fail_th, raise_th) in thresholds.items():
        # Apply TH-Fail condition (if column value is less than the fail threshold)
        aggregated_df = aggregated_df.withColumn(
            f"{col_name}_THRESHOLD_FAIL", 
            f"{col_name}_THRESHOLD_FALL", 
            when(col(col_name) < fail_th, True).otherwise(False)
        )
        # Apply TH-RAISE condition (if column value is greater than the raise threshold)
@@ -128,11 +128,11 @@ def SparkStreamer(key, kpi_list, oper_list, thresholds, stop_event,

        # --- This will write output to Kafka: ACTUAL IMPLEMENTATION
        query = thresholded_stream_data \
            .selectExpr(f"'{key}' AS key", "to_json(struct(*)) AS value") \
            .selectExpr(f"CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \
            .writeStream \
            .format("kafka") \
            .option("kafka.bootstrap.servers", KafkaConfig.get_kafka_address()) \
            .option("topic",                   KafkaTopic.ANALYTICS_RESPONSE.value) \
            .option("topic",                   KafkaTopic.ALARMS.value) \
            .option("checkpointLocation",      "analytics/.spark/checkpoint") \
            .outputMode("update")

+39 −0
Changes for src/analytics/backend/tests/messages.py: 39 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -12,6 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid
import json
from common.proto.kpi_manager_pb2        import KpiId
from common.proto.analytics_frontend_pb2 import ( AnalyzerOperationMode,
                                                Analyzer )

def get_kpi_id_list():
    return ["6e22f180-ba28-4641-b190-2287bf448888", "1e22f180-ba28-4641-b190-2287bf446666"]
@@ -32,3 +37,37 @@ def get_threshold_dict():
    return {
        op + '_value': threshold_dict[op+'_value'] for op in get_operation_list() if op + '_value' in threshold_dict
    }


def create_analyzer():
    _create_analyzer                              = Analyzer()
    # _create_analyzer.analyzer_id.analyzer_id.uuid = str(uuid.uuid4())
    _create_analyzer.analyzer_id.analyzer_id.uuid = "efef4d95-1cf1-43c4-9742-95c283ddd7a6"
    _create_analyzer.algorithm_name               = "Test_Aggergate_and_Threshold"
    _create_analyzer.operation_mode               = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING
    
    _kpi_id = KpiId()
    # input IDs to analyze
    _kpi_id.kpi_id.uuid              = str(uuid.uuid4())
    _kpi_id.kpi_id.uuid              = "6e22f180-ba28-4641-b190-2287bf448888"
    _create_analyzer.input_kpi_ids.append(_kpi_id)
    _kpi_id.kpi_id.uuid              = str(uuid.uuid4())
    _kpi_id.kpi_id.uuid              = "1e22f180-ba28-4641-b190-2287bf446666"
    _create_analyzer.input_kpi_ids.append(_kpi_id)
    _kpi_id.kpi_id.uuid              = str(uuid.uuid4())
    _create_analyzer.input_kpi_ids.append(_kpi_id)
    # output IDs after analysis
    _kpi_id.kpi_id.uuid              = str(uuid.uuid4())
    _create_analyzer.output_kpi_ids.append(_kpi_id)
    _kpi_id.kpi_id.uuid              = str(uuid.uuid4())
    _create_analyzer.output_kpi_ids.append(_kpi_id)
    # parameter
    _threshold_dict = {
        # 'avg_value'   :(20, 30), 'min_value'   :(00, 10), 'max_value'   :(45, 50),
        'first_value' :(00, 10), 'last_value'  :(40, 50), 'stdev_value':(00, 10)}
    _create_analyzer.parameters['thresholds']      = json.dumps(_threshold_dict)
    _create_analyzer.parameters['window_size']     = "60 seconds"     # Such as "10 seconds", "2 minutes", "3 hours", "4 days" or "5 weeks" 
    _create_analyzer.parameters['window_slider']   = "30 seconds"     # should be less than window size
    _create_analyzer.parameters['store_aggregate'] = str(False)       # TRUE to store. No implemented yet

    return _create_analyzer
 No newline at end of file
Loading
Loading