(CTTC) New Analytics Component

Closes #188 (closed)

Merge request reports

Loading
+1 −1
Changes for deploy/all.sh: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice n
#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"

# Uncomment to activate Monitoring Framework (new)
#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api"
#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics"

# Uncomment to activate BGP-LS Speaker
#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker"
+16 −4
Changes for deploy/tfs.sh: 16 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -182,7 +182,19 @@ kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --
    --from-literal=CRDB_SSLMODE=require
printf "\n"

echo "Create secret with Apache Kafka data for KPI and Telemetry microservices"
echo "Create secret with CockroachDB data for Analytics microservices"
CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
CRDB_DATABASE_ANALYTICS="tfs_analytics"  # TODO: change by specific configurable environment variable
kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_ANALYTICS} \
    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
    --from-literal=CRDB_SSLMODE=require
printf "\n"

echo "Create secret with Apache Kafka data for KPI, Telemetry and Analytics microservices"
KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')
kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
    --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
@@ -264,7 +276,7 @@ for COMPONENT in $TFS_COMPONENTS; do

        if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then
            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
        elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then
        elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then
            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
            $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"

@@ -287,7 +299,7 @@ for COMPONENT in $TFS_COMPONENTS; do

        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."

        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then
        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then
            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
@@ -338,7 +350,7 @@ for COMPONENT in $TFS_COMPONENTS; do
        cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"
    fi

    if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then
    if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then
        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4)
        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
+1 −1
Changes for manifests/kafka/02-kafka.yaml: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ spec:
        - name: KAFKA_LISTENERS
          value: PLAINTEXT://:9092
        - name: KAFKA_ADVERTISED_LISTENERS
          value: PLAINTEXT://localhost:9092
          value: PLAINTEXT://kafka-service.kafka.svc.cluster.local:9092
        image: wurstmeister/kafka
        imagePullPolicy: IfNotPresent
        name: kafka-broker
+128 −0
Changes for manifests/analyticsservice.yaml: 128 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: analyticsservice
spec:
  selector:
    matchLabels:
      app: analyticsservice
  #replicas: 1
  template:
    metadata:
      labels:
        app: analyticsservice
    spec:
      terminationGracePeriodSeconds: 5
      containers:
        - name: frontend
          image: labs.etsi.org:5050/tfs/controller/analytics-frontend:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 30080
            - containerPort: 9192
          env:
            - name: LOG_LEVEL
              value: "INFO"
          envFrom:
            - secretRef:
                name: crdb-analytics
            - secretRef:
                name: kfk-kpi-data
          readinessProbe:
            exec:
              command: ["/bin/grpc_health_probe", "-addr=:30080"]
          livenessProbe:
            exec:
              command: ["/bin/grpc_health_probe", "-addr=:30080"]
          resources:
            requests:
              cpu: 250m
              memory: 128Mi
            limits:
              cpu: 1000m
              memory: 1024Mi
        - name: backend
          image: labs.etsi.org:5050/tfs/controller/analytics-backend:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 30090
            - containerPort: 9192
          env:
            - name: LOG_LEVEL
              value: "INFO"
          envFrom:
            - secretRef:
                name: kfk-kpi-data
          readinessProbe:
            exec:
              command: ["/bin/grpc_health_probe", "-addr=:30090"]
          livenessProbe:
            exec:
              command: ["/bin/grpc_health_probe", "-addr=:30090"]
          resources:
            requests:
              cpu: 250m
              memory: 128Mi
            limits:
              cpu: 1000m
              memory: 1024Mi
---
apiVersion: v1
kind: Service
metadata:
  name: analyticsservice
  labels:
    app: analyticsservice
spec:
  type: ClusterIP
  selector:
    app: analyticsservice
  ports:
    - name: frontend-grpc
      protocol: TCP
      port: 30080
      targetPort: 30080
    - name: backend-grpc
      protocol: TCP
      port: 30090
      targetPort: 30090
    - name: metrics
      protocol: TCP
      port: 9192
      targetPort: 9192
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: analyticsservice-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: analyticsservice
  minReplicas: 1
  maxReplicas: 20
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 80
  #behavior:
  #  scaleDown:
  #    stabilizationWindowSeconds: 30
+15 −11
Changes for proto/analytics_frontend.proto: 15 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -30,21 +30,25 @@ message AnalyzerId {
}

enum AnalyzerOperationMode {
  ANALYZEROPERATIONMODE_BATCH     = 0;
  ANALYZEROPERATIONMODE_STREAMING = 1;
  ANALYZEROPERATIONMODE_UNSPECIFIED = 0;
  ANALYZEROPERATIONMODE_BATCH       = 1;
  ANALYZEROPERATIONMODE_STREAMING   = 2;
}

// duration field may be added in analyzer... 
message Analyzer {
  string                     algorithm_name       = 1; // The algorithm to be executed
  repeated kpi_manager.KpiId input_kpi_ids        = 2; // The KPI Ids to be processed by the analyzer
  repeated kpi_manager.KpiId output_kpi_ids       = 3; // The KPI Ids produced by the analyzer
  AnalyzerOperationMode      operation_mode       = 4; // Operation mode of the analyzer

  AnalyzerId                 analyzer_id          = 1;
  string                     algorithm_name       = 2;  // The algorithm to be executed
  float                      duration_s           = 3;  // Termiate the data analytics thread after duration (seconds); 0 = infinity time
  repeated kpi_manager.KpiId input_kpi_ids        = 4;  // The KPI Ids to be processed by the analyzer
  repeated kpi_manager.KpiId output_kpi_ids       = 5;  // The KPI Ids produced by the analyzer
  AnalyzerOperationMode      operation_mode       = 6;  // Operation mode of the analyzer
  map<string, string>        parameters           = 7;  // Add dictionary of (key, value) pairs such as (window_size, 10) etc.
  // In batch mode... 
  float                      batch_min_duration_s = 5; // ..., min duration to collect before executing batch
  float                      batch_max_duration_s = 6; // ..., max duration collected to execute the batch
  uint64                     batch_min_size       = 7; // ..., min number of samples to collect before executing batch
  uint64                     batch_max_size       = 8; // ..., max number of samples collected to execute the batch
  float                      batch_min_duration_s = 8;  // ..., min duration to collect before executing batch
  float                      batch_max_duration_s = 9;  // ..., max duration collected to execute the batch
  uint64                     batch_min_size       = 10; // ..., min number of samples to collect before executing batch
  uint64                     batch_max_size       = 11; // ..., max number of samples collected to execute the batch
}

message AnalyzerFilter {
Loading
Loading