Resolve "(CTTC) Multiple enhancements in Telemetry component and deployment logic"

Waleed Akbarrequested to merge
feat/338-cttc-multiple-enhancements-in-telemetry-component-and-deployment-logic into develop

Closes #338 (closed)

Merge request reports

proto/kpi_sample_types.proto

+0 −1
Changes for proto/kpi_sample_types.proto: 0 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -31,7 +31,6 @@ enum KpiSampleType { 
    KPISAMPLETYPE_ML_CONFIDENCE                 = 401;  //. can be used by both optical and L3 without any issue



    KPISAMPLETYPE_OPTICAL_SECURITY_STATUS       = 501;  //. can be used by both optical and L3 without any issue

    KPISAMPLETYPE_OPTICAL_POWER_TOTAL_INPUT     = 502;

    KPISAMPLETYPE_OPTICAL_TOTAL_INPUT_POWER     = 503;



    KPISAMPLETYPE_L3_UNIQUE_ATTACK_CONNS        = 601;

scripts/run_tests_locally-telemetry-backend.sh

+35 −2
Changes for scripts/run_tests_locally-telemetry-backend.sh: 35 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -19,11 +19,44 @@ PROJECTDIR=`pwd` 
cd $PROJECTDIR/src

# RCFILE=$PROJECTDIR/coverage/.coveragerc



export KFK_SERVER_ADDRESS='127.0.0.1:9092'

CRDB_SQL_ADDRESS=$(kubectl get service cockroachdb-public --namespace crdb -o jsonpath='{.spec.clusterIP}')

export CRDB_URI="cockroachdb://tfs:tfs123@${CRDB_SQL_ADDRESS}:26257/tfs_telemetry?sslmode=require"

RCFILE=$PROJECTDIR/coverage/.coveragerc

# RCFILE=$PROJECTDIR/coverage/.coveragerc



export KFK_SERVER_ADDRESS='127.0.0.1:9094'



IP_KPI=$(kubectl get all --all-namespaces | grep service/kpi-managerservice | awk '{print $4}')

export IP_KPI

echo "KPI Manager Service IP: ${IP_KPI}"



IP_TELE=$(kubectl get all --all-namespaces | grep service/telemetryservice | awk '{print $4}')

export IP_TELE

echo "Telemetry Frontend Service IP: ${IP_TELE}"



IP_CONTEXT=$(kubectl get all --all-namespaces | grep service/contextservice | awk '{print $4}')

export IP_CONTEXT

echo "Context Service IP: ${IP_CONTEXT}"



# Start Kafka port-forward in background

kubectl port-forward -n kafka service/kafka-public 9094:9094 > /dev/null 2>&1 &

KAFKA_PF_PID=$!



# Function to cleanup port-forward on exit

cleanup() {

    # echo "Cleaning up Kafka port-forward (PID: ${KAFKA_PF_PID})..."

    kill ${KAFKA_PF_PID} 2>/dev/null || true

    wait ${KAFKA_PF_PID} 2>/dev/null || true

    sleep 1

}



trap cleanup EXIT INT TERM

echo "Waiting for Kafka port-forward to be ready..."

sleep 1



# Verify port-forward is working

if ! nc -z 127.0.0.1 9094 2>/dev/null; then

    echo "WARNING: Kafka port-forward may not be ready yet"

fi



python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \

    telemetry/backend/tests/test_backend.py

scripts/run_tests_locally-telemetry-gnmi.sh

+13 −0
Changes for scripts/run_tests_locally-telemetry-gnmi.sh: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -19,6 +19,19 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc 


export KFK_SERVER_ADDRESS='127.0.0.1:9092'





IP_KPI=$(kubectl get all --all-namespaces | grep service/kpi-managerservice | awk '{print $4}')

export IP_KPI

echo "KPI Manager Service IP: ${IP_KPI}"



IP_TELE=$(kubectl get all --all-namespaces | grep service/telemetryservice | awk '{print $4}')

export IP_TELE

echo "Telemetry Frontend Service IP: ${IP_TELE}"



IP_CONTEXT=$(kubectl get all --all-namespaces | grep service/contextservice | awk '{print $4}')

export IP_CONTEXT

echo "Context Service IP: ${IP_CONTEXT}"



# This is unit test (should be tested with container-lab running)

python3 -m pytest --log-level=info --log-cli-level=info --verbose \

    telemetry/backend/tests/gnmi_oc/test_unit_GnmiOpenConfigCollector.py::test_full_workflow

src/telemetry/backend/service/collectors/gnmi_oc/GnmiOpenConfigCollector.py

+7 −1
Changes for src/telemetry/backend/service/collectors/gnmi_oc/GnmiOpenConfigCollector.py: 7 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -69,7 +69,10 @@ class GNMIOpenConfigCollector(_Collector): 
                skip_verify = self.skip_verify            # Skip TLS certificate verification (like gnmi_subscribe_example.py)

            )

            # self.logger.info("Connecting to gNMI target %s:%s with %s and %s", self.address, self.port, self.username, self.password)

            self.client.connect()           # type: ignore

            if self.client is None:

                self.logger.error("Failed to create gNMI client for target %s:%s", self.address, self.port)

                return False

            self.client.connect()           # TO be enable in test. type: ignore

            self.connected = True

            self.logger.info("Connected to gNMI target %s:%s", self.address, self.port)

            return True
@@ -134,6 +137,7 @@ class GNMIOpenConfigCollector(_Collector): 
                                resource = sub_endpoint['resource'],

                )

                self.logger.debug("Built %d candidate path(s) for endpoint '%s'", len(paths), paths)

                if self.connected and self.client:

                    self._subscriptions[sub_id] = Subscription(

                        sub_id                = sub_id,

                        gnmi_client           = self.client,                   # type: ignore
@@ -146,6 +150,8 @@ class GNMIOpenConfigCollector(_Collector): 
                    )

                    self.logger.info("Subscribing to %s with job_id %s ...", sub_endpoint, sub_id)

                    response.append(True)

                else:

                    raise ConnectionError("Not connected to gNMI target.")

            except:

                self.logger.exception("Invalid subscription format: %s", subscription)

                response.append(False)

src/telemetry/backend/service/collectors/gnmi_oc/KPI.py

+1 −1
Changes for src/telemetry/backend/service/collectors/gnmi_oc/KPI.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -26,5 +26,5 @@ class KPI(IntEnum): 
    KPISAMPLETYPE_BYTES_RECEIVED            = 202

    KPISAMPLETYPE_BYTES_DROPPED             = 203

    KPISAMPLETYPE_INBAND_POWER              = 301

    KPISAMPLETYPE_OPTICAL_TOTAL_INPUT_POWER = 302

    KPISAMPLETYPE_OPTICAL_TOTAL_INPUT_POWER = 503

    # TODO: Add more KPIs as needed,