Commit 71a27ab7 authored by Waleed Akbar's avatar Waleed Akbar
Browse files

Changes in Telemetry 

- Created the CRDB secret (`crdb-telemetry`).
- Updated the IF conditions in the `$COMPONENT` deployment loop within the `tfs.sh` file according to telemetry service requirements.
- Added the `Telemetryservice.yaml` file to the manifest folder.
- Updated the CRDB URL in `TelemetryEngine.py`.
- Made a minor formatting change in `TelemetryModel.py`.
parent 88f5d25b

deploy/tfs.sh

+21 −7
Original line number Diff line number Diff line
@@ -170,7 +170,19 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t 
    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices"

echo "Create secret with CockroachDB data for Telemetry microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_TELEMETRY="tfs_telemetry"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with Apache Kafka data for KPI and Telemetry microservices"

KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')

kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
@@ -252,15 +264,17 @@ for COMPONENT in $TFS_COMPONENTS; do 


        if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then

            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"

        elif [ "$COMPONENT" == "pathcomp" ]; then

        elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

            $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"



            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

            $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"

            if [ "$COMPONENT" == "pathcomp" ]; then

                # next command is redundant, but helpful to keep cache updated between rebuilds

                IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"

                $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

            fi

        elif [ "$COMPONENT" == "dlt" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"

            $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
@@ -273,7 +287,7 @@ for COMPONENT in $TFS_COMPONENTS; do 


        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."



        if [ "$COMPONENT" == "pathcomp" ]; then

        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
@@ -324,7 +338,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
        cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"

    fi



    if [ "$COMPONENT" == "pathcomp" ]; then

    if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then

        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

manifests/telemetryservice.yaml

0 → 100644
+128 −0
Original line number Diff line number Diff line 
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



apiVersion: apps/v1

kind: Deployment

metadata:

  name: telemetryservice

spec:

  selector:

    matchLabels:

      app: telemetryservice

  #replicas: 1

  template:

    metadata:

      labels:

        app: telemetryservice

    spec:

      terminationGracePeriodSeconds: 5

      containers:

        - name: frontend

          image: labs.etsi.org:5050/tfs/controller/telemetry-frontend:latest

          imagePullPolicy: Always

          ports:

            - containerPort: 30050

            - containerPort: 9192

          env:

            - name: LOG_LEVEL

              value: "INFO"

          envFrom:

            - secretRef:

                name: crdb-telemetry

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30050"]

          livenessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30050"]

          resources:

            requests:

              cpu: 250m

              memory: 128Mi

            limits:

              cpu: 1000m

              memory: 1024Mi

        - name: backend

          image: labs.etsi.org:5050/tfs/controller/telemetry-backend:latest

          imagePullPolicy: Always

          ports:

            - containerPort: 30060

            - containerPort: 9192

          env:

            - name: LOG_LEVEL

              value: "INFO"

          envFrom:

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30060"]

          livenessProbe:

            exec:

              command: ["/bin/grpc_health_probe", "-addr=:30060"]

          resources:

            requests:

              cpu: 250m

              memory: 128Mi

            limits:

              cpu: 1000m

              memory: 1024Mi

---

apiVersion: v1

kind: Service

metadata:

  name: telemetryservice

  labels:

    app: telemetryservice

spec:

  type: ClusterIP

  selector:

    app: telemetryservice

  ports:

    - name: frontend-grpc

      protocol: TCP

      port: 30050

      targetPort: 30050

    - name: backend-grpc

      protocol: TCP

      port: 30060

      targetPort: 30060

    - name: metrics

      protocol: TCP

      port: 9192

      targetPort: 9192

---

apiVersion: autoscaling/v2

kind: HorizontalPodAutoscaler

metadata:

  name: telemetryservice-hpa

spec:

  scaleTargetRef:

    apiVersion: apps/v1

    kind: Deployment

    name: telemetryservice

  minReplicas: 1

  maxReplicas: 20

  metrics:

    - type: Resource

      resource:

        name: cpu

        target:

          type: Utilization

          averageUtilization: 80

  #behavior:

  #  scaleDown:

  #    stabilizationWindowSeconds: 30

src/telemetry/database/TelemetryEngine.py

+3 −5
Original line number Diff line number Diff line
@@ -17,8 +17,8 @@ from common.Settings import get_setting 


LOGGER = logging.getLogger(__name__)



CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}'

# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}'

# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}'

CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}'



class TelemetryEngine:

    @staticmethod
@@ -32,7 +32,7 @@ class TelemetryEngine: 
            CRDB_PASSWORD  = "tfs123"

            CRDB_SSLMODE   = "require"

            crdb_uri = CRDB_URI_TEMPLATE.format(

                    CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE)

                    CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE)

        try:

            engine = sqlalchemy.create_engine(crdb_uri, echo=False)

            LOGGER.info(' TelemetryDB initalized with DB URL: {:}'.format(crdb_uri))
@@ -40,5 +40,3 @@ class TelemetryEngine: 
            LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri)))

            return None # type: ignore

        return engine # type: ignore

src/telemetry/database/TelemetryModel.py

+1 −1

File changed.

Contains only whitespace changes.