From 5eed4743c3195f42e677f9818604fed03c3a0a69 Mon Sep 17 00:00:00 2001
From: Waleed Akbar <wakbar@cttc.es>
Date: Fri, 6 Sep 2024 09:48:02 +0000
Subject: [PATCH] Changes in Analytics.
- UNSPECIFIED option added in the "AnalyzerOperationMode" enum as a best practice.
- In SparkStreamer, changed the thresholds parameter from optional to compulsory.
---
proto/analytics_frontend.proto | 5 +++--
src/analytics/backend/service/SparkStreaming.py | 3 +--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto
index bd28a21bf..bc7420d54 100644
--- a/proto/analytics_frontend.proto
+++ b/proto/analytics_frontend.proto
@@ -30,8 +30,9 @@ message AnalyzerId {
}
enum AnalyzerOperationMode {
- ANALYZEROPERATIONMODE_BATCH = 0;
- ANALYZEROPERATIONMODE_STREAMING = 1;
+ ANALYZEROPERATIONMODE_UNSPECIFIED = 0;
+ ANALYZEROPERATIONMODE_BATCH = 1;
+ ANALYZEROPERATIONMODE_STREAMING = 2;
}
message Analyzer {
diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py
index 73aa75025..175222b59 100644
--- a/src/analytics/backend/service/SparkStreaming.py
+++ b/src/analytics/backend/service/SparkStreaming.py
@@ -73,7 +73,7 @@ def ApplyThresholds(aggregated_df, thresholds):
)
return aggregated_df
-def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None, thresholds=None, time_stamp_col=None):
+def SparkStreamer(kpi_list, oper_list, thresholds, window_size=None, win_slide_duration=None, time_stamp_col=None):
"""
Method to perform Spark operation Kafka stream.
NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session.
@@ -86,7 +86,6 @@ def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None
if window_size is None: window_size = "60 seconds" # default
if win_slide_duration is None: win_slide_duration = "30 seconds" # default
if time_stamp_col is None: time_stamp_col = "time_stamp" # default
- if thresholds is None: thresholds = {} # No threshold will be applied
try:
# Read data from Kafka
--
GitLab