Commit f3ea42c1 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/179-re-designing-of-telemetry' into 'develop' 

Resolve "Re-designing of Telemetry"

See merge request !258
parents f09c2f45 32d9a97b

.gitlab-ci.yml

+1 −1
Original line number Diff line number Diff line
@@ -48,6 +48,6 @@ include: 
  - local: '/src/kpi_manager/.gitlab-ci.yml'

  - local: '/src/kpi_value_api/.gitlab-ci.yml'

  - local: '/src/kpi_value_writer/.gitlab-ci.yml'



  - local: '/src/telemetry/.gitlab-ci.yml'

  # This should be last one: end-to-end integration tests

  - local: '/src/tests/.gitlab-ci.yml'

deploy/kafka.sh

+4 −2
Original line number Diff line number Diff line
@@ -79,10 +79,12 @@ function kafka_deploy() { 
echo "Apache Kafka"

echo ">>> Checking if Apache Kafka is deployed ... "

if [ "$KFK_REDEPLOY" == "YES" ]; then

    echo ">>> Redeploying kafka namespace"

    kafka_deploy

elif kubectl get --namespace ${KFK_NAMESPACE} deployments.apps &> /dev/null; then

elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then

    echo ">>> Apache Kafka already present; skipping step." 

else

    echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace"

    kafka_deploy

fi

echo

deploy/tfs.sh

+21 −7
Original line number Diff line number Diff line
@@ -170,7 +170,19 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t 
    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices"

echo "Create secret with CockroachDB data for Telemetry microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_TELEMETRY="tfs_telemetry"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with Apache Kafka data for KPI and Telemetry microservices"

KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')

kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
@@ -252,15 +264,17 @@ for COMPONENT in $TFS_COMPONENTS; do 


        if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then

            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"

        elif [ "$COMPONENT" == "pathcomp" ]; then

        elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

            $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"



            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

            $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"

            if [ "$COMPONENT" == "pathcomp" ]; then

                # next command is redundant, but helpful to keep cache updated between rebuilds

                IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"

                $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

            fi

        elif [ "$COMPONENT" == "dlt" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"

            $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
@@ -273,7 +287,7 @@ for COMPONENT in $TFS_COMPONENTS; do 


        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."



        if [ "$COMPONENT" == "pathcomp" ]; then

        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
@@ -324,7 +338,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
        cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"

    fi



    if [ "$COMPONENT" == "pathcomp" ]; then

    if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

manifests/telemetryservice.yaml

0 → 100644
+128 −0
Original line number Diff line number Diff line 
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



apiVersion: apps/v1

kind: Deployment

metadata:

  name: telemetryservice

spec:

  selector:

    matchLabels:

      app: telemetryservice

  #replicas: 1

  template:

    metadata:

      labels:

        app: telemetryservice

    spec:

      terminationGracePeriodSeconds: 5

      containers:

        - name: frontend

          image: labs.etsi.org:5050/tfs/controller/telemetry-frontend:latest

          imagePullPolicy: Always

          ports:

            - containerPort: 30050

            - containerPort: 9192

          env:

            - name: LOG_LEVEL

              value: "INFO"

          envFrom:

            - secretRef:

                name: crdb-telemetry

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30050"]

          livenessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30050"]

          resources:

            requests:

              cpu: 250m

              memory: 128Mi

            limits:

              cpu: 1000m

              memory: 1024Mi

        - name: backend

          image: labs.etsi.org:5050/tfs/controller/telemetry-backend:latest

          imagePullPolicy: Always

          ports:

            - containerPort: 30060

            - containerPort: 9192

          env:

            - name: LOG_LEVEL

              value: "INFO"

          envFrom:

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30060"]

          livenessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30060"]

          resources:

            requests:

              cpu: 250m

              memory: 128Mi

            limits:

              cpu: 1000m

              memory: 1024Mi

---

apiVersion: v1

kind: Service

metadata:

  name: telemetryservice

  labels:

    app: telemetryservice

spec:

  type: ClusterIP

  selector:

    app: telemetryservice

  ports:

    - name: frontend-grpc

      protocol: TCP

      port: 30050

      targetPort: 30050

    - name: backend-grpc

      protocol: TCP

      port: 30060

      targetPort: 30060

    - name: metrics

      protocol: TCP

      port: 9192

      targetPort: 9192

---

apiVersion: autoscaling/v2

kind: HorizontalPodAutoscaler

metadata:

  name: telemetryservice-hpa

spec:

  scaleTargetRef:

    apiVersion: apps/v1

    kind: Deployment

    name: telemetryservice

  minReplicas: 1

  maxReplicas: 20

  metrics:

    - type: Resource

      resource:

        name: cpu

        target:

          type: Utilization

          averageUtilization: 80

  #behavior:

  #  scaleDown:

  #    stabilizationWindowSeconds: 30

proto/telemetry_frontend.proto

+9 −7
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ message Collector { 
  kpi_manager.KpiId  kpi_id       = 2; // The KPI Id to be associated to the collected samples

  float              duration_s   = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely

  float              interval_s   = 4; // Interval between collected samples

  context.Timestamp  start_time   = 5; // Timestamp when Collector start execution

  context.Timestamp  end_time     = 6; // Timestamp when Collector stop execution

}



message CollectorFilter {