Merge branch 'feat/192-cttc-implement-telemetry-backend-collector-gnmi-openconfig' into 'develop' (ff14c8ca) · Commits · TFS / controller

deploy/all.sh

+2 −2

Original line number	Diff line number	Diff line
		@@ -235,8 +235,8 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
		# Deploy Apache Kafka
		./deploy/kafka.sh

		#Deploy Monitoring (Prometheus, Mimir, Grafana)
		./deploy/monitoring.sh
		#Deploy Monitoring (Prometheus Gateway, Prometheus)
		# ./deploy/monitoring.sh

		# Expose Dashboard
		./deploy/expose_dashboard.sh

manifests/kafka/single-node.yaml

+10 −2

Original line number	Diff line number	Diff line
		@@ -35,6 +35,10 @@ spec:
		port: 9093
		protocol: TCP
		targetPort: 9093
		- name: external # for testing purposes (To expose Kafka outside the cluster)
		port: 9094
		protocol: TCP
		targetPort: 9094
		---
		apiVersion: apps/v1
		kind: StatefulSet
		@@ -67,15 +71,19 @@ spec:
		containerPort: 9092
		- name: control-plane
		containerPort: 9093
		- name: external # for testing purposes
		containerPort: 9094
		env:
		- name: KAFKA_CFG_NODE_ID
		value: "1"
		- name: KAFKA_CFG_PROCESS_ROLES
		value: "controller,broker"
		- name: KAFKA_CFG_LISTENERS
		value: "PLAINTEXT://:9092,CONTROLLER://:9093"
		value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" # EXTERNAL://:9094 for testing purposes
		- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
		value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
		value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT"
		- name: KAFKA_CFG_ADVERTISED_LISTENERS
		value: "PLAINTEXT://kafka-public.kafka.svc.cluster.local:9092,EXTERNAL://localhost:9094"
		- name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES
		value: "CONTROLLER"
		- name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS

scripts/run_tests_locally-kpi-prom-writer.sh

+1 −2

Original line number	Diff line number	Diff line
		@@ -19,7 +19,6 @@ PROJECTDIR=`pwd`
		cd $PROJECTDIR/src

		RCFILE=$PROJECTDIR/coverage/.coveragerc
		CRDB_SQL_ADDRESS=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.clusterIP}')
		export CRDB_URI="cockroachdb://tfs:tfs123@${CRDB_SQL_ADDRESS}:26257/tfs_kpi_mgmt?sslmode=require"

		python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \
		kpi_value_writer/tests/test_metric_writer_to_prom.py

scripts/run_tests_locally-telemetry-gnmi.sh

0 → 100755

+30 −0

Original line number	Diff line number	Diff line
		#!/bin/bash
		# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		PROJECTDIR=`pwd`
		cd $PROJECTDIR/src
		RCFILE=$PROJECTDIR/coverage/.coveragerc

		export KFK_SERVER_ADDRESS='127.0.0.1:9094'

		# This is unit test (should be tested with container-lab running)
		python3 -m pytest --log-level=info --log-cli-level=info --verbose \
		telemetry/backend/tests/gnmi_oc/test_unit_GnmiOpenConfigCollector.py

		# This is integration test (should be tested with container-lab running)
		python3 -m pytest --log-level=info --log-cli-level=info --verbose \
		telemetry/backend/tests/gnmi_oc/test_integration_GnmiOCcollector.py

		echo "Bye!"

src/common/tools/kafka/Variables.py

+9 −12

Original line number	Diff line number	Diff line
		@@ -14,7 +14,6 @@

		import logging, time
		from enum import Enum
		#from confluent_kafka.admin import AdminClient, NewTopic
		from kafka.admin import KafkaAdminClient, NewTopic
		from kafka.errors import TopicAlreadyExistsError
		from common.Settings import get_setting
		@@ -36,15 +35,14 @@ class KafkaConfig(Enum):
		def get_kafka_address() -> str:
		kafka_server_address = get_setting('KFK_SERVER_ADDRESS', default=None)
		if kafka_server_address is None:
		KFK_NAMESPACE = get_setting('KFK_NAMESPACE')
		KFK_PORT = get_setting('KFK_SERVER_PORT')
		KFK_NAMESPACE = get_setting('KFK_NAMESPACE', default='kafka')
		KFK_PORT = get_setting('KFK_SERVER_PORT', default='9092')
		kafka_server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT)
		return kafka_server_address

		@staticmethod
		def get_admin_client():
		SERVER_ADDRESS = KafkaConfig.get_kafka_address()
		#ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS})
		ADMIN_CLIENT = KafkaAdminClient(bootstrap_servers=SERVER_ADDRESS)
		return ADMIN_CLIENT

		@@ -53,10 +51,10 @@ class KafkaTopic(Enum):
		# TODO: Later to be populated from ENV variable.
		TELEMETRY_REQUEST = 'topic_telemetry_request'
		TELEMETRY_RESPONSE = 'topic_telemetry_response'
		RAW = 'topic_raw'
		LABELED = 'topic_labeled'
		VALUE = 'topic_value'
		ALARMS = 'topic_alarms'
		RAW = 'topic_raw' # TODO: Update name to telemetry_raw
		LABELED = 'topic_labeled' # TODO: Update name to telemetry_labeled
		VALUE = 'topic_value' # TODO: Update name to telemetry_value
		ALARMS = 'topic_alarms' # TODO: Update name to telemetry_alarms
		ANALYTICS_REQUEST = 'topic_analytics_request'
		ANALYTICS_RESPONSE = 'topic_analytics_response'
		VNTMANAGER_REQUEST = 'topic_vntmanager_request'
		@@ -142,7 +140,6 @@ class KafkaTopic(Enum):
		LOGGER.debug('All topics created and available.')
		return True

		# TODO: create all topics after the deployments (Telemetry and Analytics)

		if __name__ == '__main__':
		import os